From 319860003162f67fdf7b761b8e835522aea4633c Mon Sep 17 00:00:00 2001 From: Hein Puth Date: Tue, 14 Jul 2026 16:28:06 +0200 Subject: [PATCH] feat(tools): add duplicate audit report --- internal/app/app.go | 1 + internal/mcpserver/server.go | 44 ++-- internal/tools/duplicate_audit.go | 305 +++++++++++++++++++++++++ internal/tools/duplicate_audit_test.go | 76 ++++++ 4 files changed, 408 insertions(+), 18 deletions(-) create mode 100644 internal/tools/duplicate_audit.go create mode 100644 internal/tools/duplicate_audit_test.go diff --git a/internal/app/app.go b/internal/app/app.go index 8a65214..57b0c6c 100644 --- a/internal/app/app.go +++ b/internal/app/app.go @@ -202,6 +202,7 @@ func routes(logger *slog.Logger, cfg *config.Config, info buildinfo.Info, db *st Update: tools.NewUpdateTool(db, embeddings, metadata, cfg.Capture, logger), Delete: tools.NewDeleteTool(db), Archive: tools.NewArchiveTool(db), + DuplicateAudit: tools.NewDuplicateAuditTool(db, cfg.Search, activeProjects), Projects: tools.NewProjectsTool(db, activeProjects), Version: tools.NewVersionTool(cfg.MCP.ServerName, info), Learnings: tools.NewLearningsTool(db, activeProjects, cfg.Search), diff --git a/internal/mcpserver/server.go b/internal/mcpserver/server.go index f3a127e..aca2123 100644 --- a/internal/mcpserver/server.go +++ b/internal/mcpserver/server.go @@ -19,24 +19,25 @@ const ( ) type ToolSet struct { - Version *tools.VersionTool - Capture *tools.CaptureTool - Search *tools.SearchTool - List *tools.ListTool - Stats *tools.StatsTool - Get *tools.GetTool - Update *tools.UpdateTool - Delete *tools.DeleteTool - Archive *tools.ArchiveTool - Projects *tools.ProjectsTool - Context *tools.ContextTool - Recall *tools.RecallTool - Summarize *tools.SummarizeTool - Links *tools.LinksTool - Files *tools.FilesTool - Backfill *tools.BackfillTool - Reparse *tools.ReparseMetadataTool - RetryMetadata *tools.RetryEnrichmentTool + Version *tools.VersionTool + Capture *tools.CaptureTool + Search *tools.SearchTool + List *tools.ListTool + Stats *tools.StatsTool + Get *tools.GetTool + Update *tools.UpdateTool + Delete *tools.DeleteTool + Archive *tools.ArchiveTool + DuplicateAudit *tools.DuplicateAuditTool + Projects *tools.ProjectsTool + Context *tools.ContextTool + Recall *tools.RecallTool + Summarize *tools.SummarizeTool + Links *tools.LinksTool + Files *tools.FilesTool + Backfill *tools.BackfillTool + Reparse *tools.ReparseMetadataTool + RetryMetadata *tools.RetryEnrichmentTool //Maintenance *tools.MaintenanceTool Skills *tools.SkillsTool Personas *tools.AgentPersonasTool @@ -227,6 +228,12 @@ func registerThoughtTools(server *mcp.Server, logger *slog.Logger, toolSet ToolS }, toolSet.Archive.Handle); err != nil { return err } + if err := addTool(server, logger, &mcp.Tool{ + Name: "audit_duplicates", + Description: "Dry-run duplicate audit for projects, thoughts, and metadata normalization candidates; performs no writes.", + }, toolSet.DuplicateAudit.Handle); err != nil { + return err + } if err := addTool(server, logger, &mcp.Tool{ Name: "summarize_thoughts", Description: "LLM summary of a filtered set of thoughts.", @@ -764,6 +771,7 @@ func BuildToolCatalog() []tools.ToolEntry { {Name: "update_thought", Description: "Update thought content or merge metadata.", Category: "thoughts"}, {Name: "delete_thought", Description: "Hard-delete a thought by id.", Category: "thoughts"}, {Name: "archive_thought", Description: "Archive a thought so it is hidden from default search and listing.", Category: "thoughts"}, + {Name: "audit_duplicates", Description: "Dry-run duplicate audit for exact/normalized project names, thought content, and metadata value variants. Reports candidates and recommendations; performs no writes.", Category: "admin"}, {Name: "summarize_thoughts", Description: "Produce an LLM prose summary of a filtered or searched set of thoughts.", Category: "thoughts"}, {Name: "recall_context", Description: "Recall semantically relevant and recent context for prompt injection. Combines vector similarity with recency. Falls back to full-text search when no embeddings exist.", Category: "thoughts"}, {Name: "link_thoughts", Description: "Create a typed relationship between two thoughts.", Category: "thoughts"}, diff --git a/internal/tools/duplicate_audit.go b/internal/tools/duplicate_audit.go new file mode 100644 index 0000000..f503fca --- /dev/null +++ b/internal/tools/duplicate_audit.go @@ -0,0 +1,305 @@ +package tools + +import ( + "context" + "regexp" + "sort" + "strings" + + "github.com/google/uuid" + "github.com/modelcontextprotocol/go-sdk/mcp" + + "git.warky.dev/wdevs/amcs/internal/config" + "git.warky.dev/wdevs/amcs/internal/session" + "git.warky.dev/wdevs/amcs/internal/store" + thoughttypes "git.warky.dev/wdevs/amcs/internal/types" +) + +const defaultDuplicateAuditLimit = 50 + +var duplicateWhitespace = regexp.MustCompile(`\s+`) + +type DuplicateAuditTool struct { + store *store.DB + search config.SearchConfig + sessions *session.ActiveProjects +} + +type DuplicateAuditInput struct { + Project string `json:"project,omitempty" jsonschema:"optional project name or id to scope thought duplicate scanning"` + Limit int `json:"limit,omitempty" jsonschema:"maximum candidate groups to return"` + ThoughtLimit int `json:"thought_limit,omitempty" jsonschema:"maximum thoughts to scan for duplicate content; defaults to search limit"` + IncludeArchived bool `json:"include_archived,omitempty" jsonschema:"include archived thoughts in the scan"` +} + +type DuplicateAuditOutput struct { + Report DuplicateAuditReport `json:"report"` +} + +type DuplicateAuditReport struct { + Summary DuplicateAuditSummary `json:"summary"` + ProjectCandidates []DuplicateProjectCandidate `json:"project_candidates"` + ThoughtCandidates []DuplicateThoughtCandidate `json:"thought_candidates"` + MetadataCandidates []MetadataCleanupCandidate `json:"metadata_candidates"` + RecommendedNextStep string `json:"recommended_next_step"` +} + +type DuplicateAuditSummary struct { + DryRun bool `json:"dry_run"` + ScannedProjects int `json:"scanned_projects"` + ScannedThoughts int `json:"scanned_thoughts"` + ProjectCandidateGroups int `json:"project_candidate_groups"` + ThoughtCandidateGroups int `json:"thought_candidate_groups"` + MetadataCandidateGroups int `json:"metadata_candidate_groups"` + Truncated bool `json:"truncated"` +} + +type DuplicateProjectCandidate struct { + MatchType string `json:"match_type"` + NormalizedValue string `json:"normalized_value"` + Confidence float64 `json:"confidence"` + RecommendedCanonicalID uuid.UUID `json:"recommended_canonical_id"` + Projects []thoughttypes.ProjectSummary `json:"projects"` +} + +type DuplicateThoughtCandidate struct { + MatchType string `json:"match_type"` + NormalizedValue string `json:"normalized_value"` + Confidence float64 `json:"confidence"` + RecommendedCanonicalID uuid.UUID `json:"recommended_canonical_id"` + Thoughts []thoughttypes.Thought `json:"thoughts"` +} + +type MetadataCleanupCandidate struct { + Field string `json:"field"` + NormalizedValue string `json:"normalized_value"` + Values []string `json:"values"` + Occurrences int `json:"occurrences"` + RecommendedValue string `json:"recommended_value"` + RecommendedAction string `json:"recommended_action"` +} + +func NewDuplicateAuditTool(db *store.DB, search config.SearchConfig, sessions *session.ActiveProjects) *DuplicateAuditTool { + return &DuplicateAuditTool{store: db, search: search, sessions: sessions} +} + +func (t *DuplicateAuditTool) Handle(ctx context.Context, req *mcp.CallToolRequest, in DuplicateAuditInput) (*mcp.CallToolResult, DuplicateAuditOutput, error) { + project, err := resolveProject(ctx, t.store, t.sessions, req, in.Project, false) + if err != nil { + return nil, DuplicateAuditOutput{}, err + } + var projectID *int64 + if project != nil { + projectID = &project.NumericID + } + + projects, err := t.store.ListProjects(ctx) + if err != nil { + return nil, DuplicateAuditOutput{}, err + } + + thoughtLimit := normalizeLimit(in.ThoughtLimit, t.search) + if in.ThoughtLimit <= 0 { + thoughtLimit = normalizeLimit(0, t.search) + } + thoughts, err := t.store.ListThoughts(ctx, thoughttypes.ListFilter{Limit: thoughtLimit, ProjectID: projectID, IncludeArchived: in.IncludeArchived}) + if err != nil { + return nil, DuplicateAuditOutput{}, err + } + + if project != nil { + _ = t.store.TouchProject(ctx, project.NumericID) + } + return nil, DuplicateAuditOutput{Report: buildDuplicateAuditReport(projects, thoughts, in)}, nil +} + +func buildDuplicateAuditReport(projects []thoughttypes.ProjectSummary, thoughts []thoughttypes.Thought, in DuplicateAuditInput) DuplicateAuditReport { + limit := in.Limit + if limit <= 0 { + limit = defaultDuplicateAuditLimit + } + + projectCandidates, projectTruncated := duplicateProjectCandidates(projects, limit) + thoughtCandidates, thoughtTruncated := duplicateThoughtCandidates(thoughts, limit) + metadataCandidates, metadataTruncated := metadataCleanupCandidates(thoughts, limit) + + return DuplicateAuditReport{ + Summary: DuplicateAuditSummary{ + DryRun: true, + ScannedProjects: len(projects), + ScannedThoughts: len(thoughts), + ProjectCandidateGroups: len(projectCandidates), + ThoughtCandidateGroups: len(thoughtCandidates), + MetadataCandidateGroups: len(metadataCandidates), + Truncated: projectTruncated || thoughtTruncated || metadataTruncated, + }, + ProjectCandidates: projectCandidates, + ThoughtCandidates: thoughtCandidates, + MetadataCandidates: metadataCandidates, + RecommendedNextStep: "Review candidates manually, then use explicit archive/update/merge tools; this audit performs no writes.", + } +} + +func duplicateProjectCandidates(projects []thoughttypes.ProjectSummary, limit int) ([]DuplicateProjectCandidate, bool) { + groups := map[string][]thoughttypes.ProjectSummary{} + for _, project := range projects { + key := normalizeDuplicateText(project.Name) + if key != "" { + groups[key] = append(groups[key], project) + } + } + keys := sortedDuplicateKeys(groups) + out := make([]DuplicateProjectCandidate, 0, min(len(keys), limit)) + for _, key := range keys { + items := groups[key] + sort.SliceStable(items, func(i, j int) bool { + if items[i].ThoughtCount != items[j].ThoughtCount { + return items[i].ThoughtCount > items[j].ThoughtCount + } + return items[i].CreatedAt.Before(items[j].CreatedAt) + }) + matchType := "normalized_name" + if allProjectNamesExact(items) { + matchType = "exact_name" + } + out = append(out, DuplicateProjectCandidate{MatchType: matchType, NormalizedValue: key, Confidence: 1.0, RecommendedCanonicalID: items[0].ID, Projects: items}) + if len(out) == limit { + return out, len(keys) > limit + } + } + return out, false +} + +func duplicateThoughtCandidates(thoughts []thoughttypes.Thought, limit int) ([]DuplicateThoughtCandidate, bool) { + groups := map[string][]thoughttypes.Thought{} + for _, thought := range thoughts { + key := normalizeDuplicateText(thought.Content) + if key != "" { + groups[key] = append(groups[key], thought) + } + } + keys := sortedDuplicateKeys(groups) + out := make([]DuplicateThoughtCandidate, 0, min(len(keys), limit)) + for _, key := range keys { + items := groups[key] + sort.SliceStable(items, func(i, j int) bool { return items[i].CreatedAt.Before(items[j].CreatedAt) }) + matchType := "normalized_content" + if allThoughtContentExact(items) { + matchType = "exact_content" + } + out = append(out, DuplicateThoughtCandidate{MatchType: matchType, NormalizedValue: key, Confidence: 1.0, RecommendedCanonicalID: items[0].GUID, Thoughts: items}) + if len(out) == limit { + return out, len(keys) > limit + } + } + return out, false +} + +func metadataCleanupCandidates(thoughts []thoughttypes.Thought, limit int) ([]MetadataCleanupCandidate, bool) { + groups := map[string]map[string]int{} + add := func(field, value string) { + trimmed := strings.TrimSpace(value) + key := normalizeDuplicateText(trimmed) + if key == "" || trimmed == "" { + return + } + bucketKey := field + "\x00" + key + if groups[bucketKey] == nil { + groups[bucketKey] = map[string]int{} + } + groups[bucketKey][trimmed]++ + } + for _, thought := range thoughts { + add("type", thought.Metadata.Type) + add("source", thought.Metadata.Source) + for _, topic := range thought.Metadata.Topics { + add("topics", topic) + } + for _, person := range thought.Metadata.People { + add("people", person) + } + } + + keys := make([]string, 0, len(groups)) + for key, values := range groups { + if len(values) > 1 { + keys = append(keys, key) + } + } + sort.Strings(keys) + + out := make([]MetadataCleanupCandidate, 0, min(len(keys), limit)) + for _, key := range keys { + parts := strings.SplitN(key, "\x00", 2) + values, occurrences, recommended := valuesByCount(groups[key]) + out = append(out, MetadataCleanupCandidate{ + Field: parts[0], + NormalizedValue: parts[1], + Values: values, + Occurrences: occurrences, + RecommendedValue: recommended, + RecommendedAction: "preview bulk metadata remap before applying; no changes made by audit", + }) + if len(out) == limit { + return out, len(keys) > limit + } + } + return out, false +} + +func sortedDuplicateKeys[T any](groups map[string][]T) []string { + keys := make([]string, 0, len(groups)) + for key, items := range groups { + if len(items) > 1 { + keys = append(keys, key) + } + } + sort.Strings(keys) + return keys +} + +func valuesByCount(counts map[string]int) ([]string, int, string) { + values := make([]string, 0, len(counts)) + occurrences := 0 + for value, count := range counts { + values = append(values, value) + occurrences += count + } + sort.Slice(values, func(i, j int) bool { + if counts[values[i]] != counts[values[j]] { + return counts[values[i]] > counts[values[j]] + } + return values[i] < values[j] + }) + return values, occurrences, values[0] +} + +func normalizeDuplicateText(value string) string { + return duplicateWhitespace.ReplaceAllString(strings.ToLower(strings.TrimSpace(value)), " ") +} + +func allProjectNamesExact(projects []thoughttypes.ProjectSummary) bool { + if len(projects) == 0 { + return false + } + first := strings.TrimSpace(projects[0].Name) + for _, project := range projects[1:] { + if strings.TrimSpace(project.Name) != first { + return false + } + } + return true +} + +func allThoughtContentExact(thoughts []thoughttypes.Thought) bool { + if len(thoughts) == 0 { + return false + } + first := strings.TrimSpace(thoughts[0].Content) + for _, thought := range thoughts[1:] { + if strings.TrimSpace(thought.Content) != first { + return false + } + } + return true +} diff --git a/internal/tools/duplicate_audit_test.go b/internal/tools/duplicate_audit_test.go new file mode 100644 index 0000000..93455d3 --- /dev/null +++ b/internal/tools/duplicate_audit_test.go @@ -0,0 +1,76 @@ +package tools + +import ( + "testing" + "time" + + "github.com/google/uuid" + + thoughttypes "git.warky.dev/wdevs/amcs/internal/types" +) + +func TestBuildDuplicateAuditGroupsProjectsAndThoughtsSafely(t *testing.T) { + now := time.Date(2026, 4, 12, 20, 5, 0, 0, time.UTC) + projects := []thoughttypes.ProjectSummary{ + {Project: thoughttypes.Project{ID: uuid.MustParse("11111111-1111-1111-1111-111111111111"), NumericID: 1, Name: "AMCS", CreatedAt: now}, ThoughtCount: 3}, + {Project: thoughttypes.Project{ID: uuid.MustParse("22222222-2222-2222-2222-222222222222"), NumericID: 2, Name: "amcs ", CreatedAt: now.Add(time.Hour)}, ThoughtCount: 1}, + {Project: thoughttypes.Project{ID: uuid.MustParse("33333333-3333-3333-3333-333333333333"), NumericID: 3, Name: "other", CreatedAt: now}, ThoughtCount: 1}, + } + thoughts := []thoughttypes.Thought{ + {ID: 10, GUID: uuid.MustParse("aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa"), Content: "Same text", Metadata: thoughttypes.ThoughtMetadata{Type: "Note", Topics: []string{"Go", "go"}, People: []string{"Alice"}}, CreatedAt: now}, + {ID: 11, GUID: uuid.MustParse("bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb"), Content: " same text ", Metadata: thoughttypes.ThoughtMetadata{Type: "note", Topics: []string{"go"}, People: []string{"alice"}}, CreatedAt: now.Add(time.Hour)}, + {ID: 12, GUID: uuid.MustParse("cccccccc-cccc-cccc-cccc-cccccccccccc"), Content: "unique", Metadata: thoughttypes.ThoughtMetadata{Type: "Task", Topics: []string{"Tasks"}}, CreatedAt: now}, + } + + report := buildDuplicateAuditReport(projects, thoughts, DuplicateAuditInput{}) + + if len(report.ProjectCandidates) != 1 { + t.Fatalf("project candidates = %d, want 1: %#v", len(report.ProjectCandidates), report.ProjectCandidates) + } + projectCandidate := report.ProjectCandidates[0] + if projectCandidate.MatchType != "normalized_name" || projectCandidate.NormalizedValue != "amcs" || len(projectCandidate.Projects) != 2 { + t.Fatalf("project candidate mismatch: %#v", projectCandidate) + } + if projectCandidate.RecommendedCanonicalID != projects[0].ID { + t.Fatalf("recommended project = %s, want %s", projectCandidate.RecommendedCanonicalID, projects[0].ID) + } + + if len(report.ThoughtCandidates) != 1 { + t.Fatalf("thought candidates = %d, want 1: %#v", len(report.ThoughtCandidates), report.ThoughtCandidates) + } + thoughtCandidate := report.ThoughtCandidates[0] + if thoughtCandidate.MatchType != "normalized_content" || thoughtCandidate.NormalizedValue != "same text" || len(thoughtCandidate.Thoughts) != 2 { + t.Fatalf("thought candidate mismatch: %#v", thoughtCandidate) + } + if thoughtCandidate.RecommendedCanonicalID != thoughts[0].GUID { + t.Fatalf("recommended thought = %s, want %s", thoughtCandidate.RecommendedCanonicalID, thoughts[0].GUID) + } + + if len(report.MetadataCandidates) != 3 { + t.Fatalf("metadata candidates = %d, want 3: %#v", len(report.MetadataCandidates), report.MetadataCandidates) + } + if report.Summary.ProjectCandidateGroups != 1 || report.Summary.ThoughtCandidateGroups != 1 || report.Summary.MetadataCandidateGroups != 3 { + t.Fatalf("summary mismatch: %#v", report.Summary) + } + if report.Summary.DryRun != true { + t.Fatalf("dry run = false, want true") + } +} + +func TestBuildDuplicateAuditRespectsLimit(t *testing.T) { + projects := []thoughttypes.ProjectSummary{ + {Project: thoughttypes.Project{ID: uuid.MustParse("11111111-1111-1111-1111-111111111111"), Name: "Alpha"}}, + {Project: thoughttypes.Project{ID: uuid.MustParse("22222222-2222-2222-2222-222222222222"), Name: "alpha"}}, + {Project: thoughttypes.Project{ID: uuid.MustParse("33333333-3333-3333-3333-333333333333"), Name: "Beta"}}, + {Project: thoughttypes.Project{ID: uuid.MustParse("44444444-4444-4444-4444-444444444444"), Name: "beta"}}, + } + + report := buildDuplicateAuditReport(projects, nil, DuplicateAuditInput{Limit: 1}) + + if len(report.ProjectCandidates) != 1 { + t.Fatalf("project candidates = %d, want 1", len(report.ProjectCandidates)) + } + if report.Summary.Truncated != true { + t.Fatalf("truncated = false, want true") + } +}