package tools import ( "context" "regexp" "sort" "strings" "github.com/google/uuid" "github.com/modelcontextprotocol/go-sdk/mcp" "git.warky.dev/wdevs/amcs/internal/config" "git.warky.dev/wdevs/amcs/internal/session" "git.warky.dev/wdevs/amcs/internal/store" thoughttypes "git.warky.dev/wdevs/amcs/internal/types" ) const defaultDuplicateAuditLimit = 50 var duplicateWhitespace = regexp.MustCompile(`\s+`) type DuplicateAuditTool struct { store *store.DB search config.SearchConfig sessions *session.ActiveProjects } type DuplicateAuditInput struct { Project string `json:"project,omitempty" jsonschema:"optional project name or id to scope thought duplicate scanning"` Limit int `json:"limit,omitempty" jsonschema:"maximum candidate groups to return"` ThoughtLimit int `json:"thought_limit,omitempty" jsonschema:"maximum thoughts to scan for duplicate content; defaults to search limit"` IncludeArchived bool `json:"include_archived,omitempty" jsonschema:"include archived thoughts in the scan"` } type DuplicateAuditOutput struct { Report DuplicateAuditReport `json:"report"` } type DuplicateAuditReport struct { Summary DuplicateAuditSummary `json:"summary"` ProjectCandidates []DuplicateProjectCandidate `json:"project_candidates"` ThoughtCandidates []DuplicateThoughtCandidate `json:"thought_candidates"` MetadataCandidates []MetadataCleanupCandidate `json:"metadata_candidates"` RecommendedNextStep string `json:"recommended_next_step"` } type DuplicateAuditSummary struct { DryRun bool `json:"dry_run"` ScannedProjects int `json:"scanned_projects"` ScannedThoughts int `json:"scanned_thoughts"` ProjectCandidateGroups int `json:"project_candidate_groups"` ThoughtCandidateGroups int `json:"thought_candidate_groups"` MetadataCandidateGroups int `json:"metadata_candidate_groups"` Truncated bool `json:"truncated"` } type DuplicateProjectCandidate struct { MatchType string `json:"match_type"` NormalizedValue string `json:"normalized_value"` Confidence float64 `json:"confidence"` RecommendedCanonicalID uuid.UUID `json:"recommended_canonical_id"` Projects []thoughttypes.ProjectSummary `json:"projects"` } type DuplicateThoughtCandidate struct { MatchType string `json:"match_type"` NormalizedValue string `json:"normalized_value"` Confidence float64 `json:"confidence"` RecommendedCanonicalID uuid.UUID `json:"recommended_canonical_id"` Thoughts []thoughttypes.Thought `json:"thoughts"` } type MetadataCleanupCandidate struct { Field string `json:"field"` NormalizedValue string `json:"normalized_value"` Values []string `json:"values"` Occurrences int `json:"occurrences"` RecommendedValue string `json:"recommended_value"` RecommendedAction string `json:"recommended_action"` } func NewDuplicateAuditTool(db *store.DB, search config.SearchConfig, sessions *session.ActiveProjects) *DuplicateAuditTool { return &DuplicateAuditTool{store: db, search: search, sessions: sessions} } func (t *DuplicateAuditTool) Handle(ctx context.Context, req *mcp.CallToolRequest, in DuplicateAuditInput) (*mcp.CallToolResult, DuplicateAuditOutput, error) { project, err := resolveProject(ctx, t.store, t.sessions, req, in.Project, false) if err != nil { return nil, DuplicateAuditOutput{}, err } var projectID *int64 if project != nil { projectID = &project.NumericID } projects, err := t.store.ListProjects(ctx) if err != nil { return nil, DuplicateAuditOutput{}, err } thoughtLimit := normalizeLimit(in.ThoughtLimit, t.search) if in.ThoughtLimit <= 0 { thoughtLimit = normalizeLimit(0, t.search) } thoughts, err := t.store.ListThoughts(ctx, thoughttypes.ListFilter{Limit: thoughtLimit, ProjectID: projectID, IncludeArchived: in.IncludeArchived}) if err != nil { return nil, DuplicateAuditOutput{}, err } if project != nil { _ = t.store.TouchProject(ctx, project.NumericID) } return nil, DuplicateAuditOutput{Report: buildDuplicateAuditReport(projects, thoughts, in)}, nil } func buildDuplicateAuditReport(projects []thoughttypes.ProjectSummary, thoughts []thoughttypes.Thought, in DuplicateAuditInput) DuplicateAuditReport { limit := in.Limit if limit <= 0 { limit = defaultDuplicateAuditLimit } projectCandidates, projectTruncated := duplicateProjectCandidates(projects, limit) thoughtCandidates, thoughtTruncated := duplicateThoughtCandidates(thoughts, limit) metadataCandidates, metadataTruncated := metadataCleanupCandidates(thoughts, limit) return DuplicateAuditReport{ Summary: DuplicateAuditSummary{ DryRun: true, ScannedProjects: len(projects), ScannedThoughts: len(thoughts), ProjectCandidateGroups: len(projectCandidates), ThoughtCandidateGroups: len(thoughtCandidates), MetadataCandidateGroups: len(metadataCandidates), Truncated: projectTruncated || thoughtTruncated || metadataTruncated, }, ProjectCandidates: projectCandidates, ThoughtCandidates: thoughtCandidates, MetadataCandidates: metadataCandidates, RecommendedNextStep: "Review candidates manually, then use explicit archive/update/merge tools; this audit performs no writes.", } } func duplicateProjectCandidates(projects []thoughttypes.ProjectSummary, limit int) ([]DuplicateProjectCandidate, bool) { groups := map[string][]thoughttypes.ProjectSummary{} for _, project := range projects { key := normalizeDuplicateText(project.Name) if key != "" { groups[key] = append(groups[key], project) } } keys := sortedDuplicateKeys(groups) out := make([]DuplicateProjectCandidate, 0, min(len(keys), limit)) for _, key := range keys { items := groups[key] sort.SliceStable(items, func(i, j int) bool { if items[i].ThoughtCount != items[j].ThoughtCount { return items[i].ThoughtCount > items[j].ThoughtCount } return items[i].CreatedAt.Before(items[j].CreatedAt) }) matchType := "normalized_name" if allProjectNamesExact(items) { matchType = "exact_name" } out = append(out, DuplicateProjectCandidate{MatchType: matchType, NormalizedValue: key, Confidence: 1.0, RecommendedCanonicalID: items[0].ID, Projects: items}) if len(out) == limit { return out, len(keys) > limit } } return out, false } func duplicateThoughtCandidates(thoughts []thoughttypes.Thought, limit int) ([]DuplicateThoughtCandidate, bool) { groups := map[string][]thoughttypes.Thought{} for _, thought := range thoughts { key := normalizeDuplicateText(thought.Content) if key != "" { groups[key] = append(groups[key], thought) } } keys := sortedDuplicateKeys(groups) out := make([]DuplicateThoughtCandidate, 0, min(len(keys), limit)) for _, key := range keys { items := groups[key] sort.SliceStable(items, func(i, j int) bool { return items[i].CreatedAt.Before(items[j].CreatedAt) }) matchType := "normalized_content" if allThoughtContentExact(items) { matchType = "exact_content" } out = append(out, DuplicateThoughtCandidate{MatchType: matchType, NormalizedValue: key, Confidence: 1.0, RecommendedCanonicalID: items[0].GUID, Thoughts: items}) if len(out) == limit { return out, len(keys) > limit } } return out, false } func metadataCleanupCandidates(thoughts []thoughttypes.Thought, limit int) ([]MetadataCleanupCandidate, bool) { groups := map[string]map[string]int{} add := func(field, value string) { trimmed := strings.TrimSpace(value) key := normalizeDuplicateText(trimmed) if key == "" || trimmed == "" { return } bucketKey := field + "\x00" + key if groups[bucketKey] == nil { groups[bucketKey] = map[string]int{} } groups[bucketKey][trimmed]++ } for _, thought := range thoughts { add("type", thought.Metadata.Type) add("source", thought.Metadata.Source) for _, topic := range thought.Metadata.Topics { add("topics", topic) } for _, person := range thought.Metadata.People { add("people", person) } } keys := make([]string, 0, len(groups)) for key, values := range groups { if len(values) > 1 { keys = append(keys, key) } } sort.Strings(keys) out := make([]MetadataCleanupCandidate, 0, min(len(keys), limit)) for _, key := range keys { parts := strings.SplitN(key, "\x00", 2) values, occurrences, recommended := valuesByCount(groups[key]) out = append(out, MetadataCleanupCandidate{ Field: parts[0], NormalizedValue: parts[1], Values: values, Occurrences: occurrences, RecommendedValue: recommended, RecommendedAction: "preview bulk metadata remap before applying; no changes made by audit", }) if len(out) == limit { return out, len(keys) > limit } } return out, false } func sortedDuplicateKeys[T any](groups map[string][]T) []string { keys := make([]string, 0, len(groups)) for key, items := range groups { if len(items) > 1 { keys = append(keys, key) } } sort.Strings(keys) return keys } func valuesByCount(counts map[string]int) ([]string, int, string) { values := make([]string, 0, len(counts)) occurrences := 0 for value, count := range counts { values = append(values, value) occurrences += count } sort.Slice(values, func(i, j int) bool { if counts[values[i]] != counts[values[j]] { return counts[values[i]] > counts[values[j]] } return values[i] < values[j] }) return values, occurrences, values[0] } func normalizeDuplicateText(value string) string { return duplicateWhitespace.ReplaceAllString(strings.ToLower(strings.TrimSpace(value)), " ") } func allProjectNamesExact(projects []thoughttypes.ProjectSummary) bool { if len(projects) == 0 { return false } first := strings.TrimSpace(projects[0].Name) for _, project := range projects[1:] { if strings.TrimSpace(project.Name) != first { return false } } return true } func allThoughtContentExact(thoughts []thoughttypes.Thought) bool { if len(thoughts) == 0 { return false } first := strings.TrimSpace(thoughts[0].Content) for _, thought := range thoughts[1:] { if strings.TrimSpace(thought.Content) != first { return false } } return true }