diff --git a/.gitea/workflows/ci.yml b/.gitea/workflows/ci.yml index 6560315..9c8332d 100644 --- a/.gitea/workflows/ci.yml +++ b/.gitea/workflows/ci.yml @@ -39,16 +39,36 @@ jobs: - name: Download dependencies run: go mod download + - name: Set up Node + uses: actions/setup-node@v4 + with: + node-version: 'lts/*' + + - name: Install pnpm + run: npm install -g pnpm + + - name: Build UI + run: | + cd ui + pnpm install --frozen-lockfile + pnpm run build + - name: Tidy modules run: go mod tidy - - name: Install UI dependencies - run: pnpm install --frozen-lockfile - working-directory: ui + - name: Set up Node + uses: actions/setup-node@v4 + with: + node-version: 'lts/*' - - name: Build UI assets - run: pnpm run build - working-directory: ui + - name: Install pnpm + run: npm install -g pnpm + + - name: Build UI + run: | + cd ui + pnpm install --frozen-lockfile + pnpm run build - name: Run tests run: go test ./... diff --git a/README.md b/README.md index 580e1af..f6c3288 100644 --- a/README.md +++ b/README.md @@ -74,6 +74,38 @@ The AMCS directory is used to store configuration and code for the Avalon Memory | `describe_tools` | List all available MCP tools with names, descriptions, categories, and model-authored usage notes; call this at the start of a session to orient yourself | | `annotate_tool` | Persist your own usage notes for a specific tool; notes are returned by `describe_tools` in future sessions | +## Webhook ingestion + +External automation can create thoughts without speaking MCP by posting JSON to `POST /webhooks/thoughts`. The endpoint is protected by the same AMCS authentication middleware as MCP and file uploads, so pass one configured API key via `x-brain-key`, an authorization bearer token header, or another enabled auth method. + +Example: + +```bash +curl -X POST http://localhost:8080/webhooks/thoughts \ + -H 'Content-Type: application/json' \ + -H 'x-brain-key: ' \ + -H 'Idempotency-Key: n8n-run-123' \ + -d '{ + "content": "External system observed build failure on main", + "project": "amcs", + "source": "n8n", + "type": "task", + "topics": ["ci", "webhook"], + "metadata": {"workflow": "ci-monitor", "run_id": "123"} + }' +``` + +Payload fields: + +- `content` is required and becomes the thought content. +- `project` is optional; when present it must match an existing AMCS project. +- `source`, `type`, `topics`, `people`, `action_items`, and `dates_mentioned` are normalized into the standard thought metadata schema. Unknown `type` values fall back to `observation`. +- `metadata` or `source_metadata` may contain safe source-specific JSON values; unsupported values and overly deep objects are dropped rather than persisted. +- `idempotency_key` or the `Idempotency-Key` header can be supplied to make repeated webhook deliveries return the existing thought with `duplicate: true`. +- `external_id` is stored under `metadata.webhook.external_id` for source-side traceability. + +Successful new ingestion returns `201` with the created thought. Duplicate idempotency-key delivery returns `200` and the previously created thought. Invalid JSON, missing content, missing/unknown projects, or unauthenticated requests are rejected before persistence. Metadata and embedding enrichment are queued after the thought is stored. + ## Learnings Learnings are curated, structured memory records for durable insights you want to keep distinct from raw thoughts. Use them for normalized lessons, decisions, and evidence-backed findings that should be easy to retrieve and review over time. diff --git a/internal/app/app.go b/internal/app/app.go index 8a65214..ed0b7b8 100644 --- a/internal/app/app.go +++ b/internal/app/app.go @@ -202,6 +202,7 @@ func routes(logger *slog.Logger, cfg *config.Config, info buildinfo.Info, db *st Update: tools.NewUpdateTool(db, embeddings, metadata, cfg.Capture, logger), Delete: tools.NewDeleteTool(db), Archive: tools.NewArchiveTool(db), + DuplicateAudit: tools.NewDuplicateAuditTool(db, cfg.Search, activeProjects), Projects: tools.NewProjectsTool(db, activeProjects), Version: tools.NewVersionTool(cfg.MCP.ServerName, info), Learnings: tools.NewLearningsTool(db, activeProjects, cfg.Search), @@ -238,6 +239,7 @@ func routes(logger *slog.Logger, cfg *config.Config, info buildinfo.Info, db *st } mux.Handle("/files", authMiddleware(fileHandler(filesTool))) mux.Handle("/files/{id}", authMiddleware(fileHandler(filesTool))) + mux.Handle("/webhooks/thoughts", authMiddleware(newWebhookThoughtHandler(db, embeddings, cfg.Capture, enrichmentRetryer, backfillTool))) mux.HandleFunc("/.well-known/oauth-authorization-server", oauthMetadataHandler()) mux.HandleFunc("/api/oauth/register", oauthRegisterHandler(dynClients, logger)) mux.HandleFunc("/api/oauth/authorize", oauthAuthorizeHandler(dynClients, authCodes, logger)) diff --git a/internal/app/webhooks.go b/internal/app/webhooks.go new file mode 100644 index 0000000..309f014 --- /dev/null +++ b/internal/app/webhooks.go @@ -0,0 +1,214 @@ +package app + +import ( + "encoding/json" + "errors" + "net/http" + "strings" + "time" + + "git.warky.dev/wdevs/amcs/internal/ai" + "git.warky.dev/wdevs/amcs/internal/config" + "git.warky.dev/wdevs/amcs/internal/metadata" + "git.warky.dev/wdevs/amcs/internal/store" + "git.warky.dev/wdevs/amcs/internal/tools" + thoughttypes "git.warky.dev/wdevs/amcs/internal/types" +) + +const maxWebhookBodyBytes = 1 << 20 + +type webhookThoughtRequest struct { + Content string `json:"content"` + Project string `json:"project,omitempty"` + Source string `json:"source,omitempty"` + Type string `json:"type,omitempty"` + Topics []string `json:"topics,omitempty"` + People []string `json:"people,omitempty"` + ActionItems []string `json:"action_items,omitempty"` + DatesMentioned []string `json:"dates_mentioned,omitempty"` + Metadata map[string]any `json:"metadata,omitempty"` + SourceMetadata map[string]any `json:"source_metadata,omitempty"` + IDempotencyKey string `json:"idempotency_key,omitempty"` + ExternalID string `json:"external_id,omitempty"` +} + +type webhookThoughtResponse struct { + Thought thoughttypes.Thought `json:"thought"` + Duplicate bool `json:"duplicate"` + WebhookMeta thoughttypes.WebhookMetadata `json:"webhook"` +} + +type webhookThoughtHandler struct { + store *store.DB + embeddings *ai.EmbeddingRunner + capture config.CaptureConfig + retryer tools.MetadataQueuer + embedRetryer tools.EmbeddingQueuer +} + +func newWebhookThoughtHandler(db *store.DB, embeddings *ai.EmbeddingRunner, capture config.CaptureConfig, retryer tools.MetadataQueuer, embedRetryer tools.EmbeddingQueuer) http.Handler { + return &webhookThoughtHandler{store: db, embeddings: embeddings, capture: capture, retryer: retryer, embedRetryer: embedRetryer} +} + +func (h *webhookThoughtHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/webhooks/thoughts" { + http.NotFound(w, r) + return + } + if r.Method != http.MethodPost { + w.Header().Set("Allow", http.MethodPost) + http.Error(w, "method not allowed", http.StatusMethodNotAllowed) + return + } + + r.Body = http.MaxBytesReader(w, r.Body, maxWebhookBodyBytes) + in, err := parseWebhookThoughtRequest(r) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + + webhookMeta := buildWebhookMetadata(in, r.Header.Get("Idempotency-Key"), time.Now().UTC()) + if webhookMeta.IDempotencyKey != "" { + if existing, err := h.store.GetThoughtByWebhookIDempotencyKey(r.Context(), webhookMeta.IDempotencyKey); err == nil { + writeWebhookThoughtResponse(w, http.StatusOK, webhookThoughtResponse{Thought: existing, Duplicate: true, WebhookMeta: webhookMeta}) + return + } + } + + projectID, err := h.resolveWebhookProject(r, in.Project) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + + thought := thoughttypes.Thought{ + Content: strings.TrimSpace(in.Content), + Metadata: normalizeWebhookThoughtMetadata(in, webhookMeta, h.capture), + ProjectID: projectID, + } + created, err := h.store.InsertThought(r.Context(), thought, h.embeddings.PrimaryModel()) + if err != nil { + http.Error(w, "insert thought: "+err.Error(), http.StatusInternalServerError) + return + } + if projectID != nil { + _ = h.store.TouchProject(r.Context(), *projectID) + } + if h.retryer != nil { + h.retryer.QueueThought(created.ID) + } + if h.embedRetryer != nil { + h.embedRetryer.QueueThought(r.Context(), created.ID, created.Content) + } + + writeWebhookThoughtResponse(w, http.StatusCreated, webhookThoughtResponse{Thought: created, WebhookMeta: webhookMeta}) +} + +func parseWebhookThoughtRequest(r *http.Request) (webhookThoughtRequest, error) { + if !strings.Contains(r.Header.Get("Content-Type"), "application/json") { + return webhookThoughtRequest{}, errors.New("webhook requires application/json") + } + defer r.Body.Close() + decoder := json.NewDecoder(r.Body) + decoder.DisallowUnknownFields() + var in webhookThoughtRequest + if err := decoder.Decode(&in); err != nil { + return webhookThoughtRequest{}, err + } + if strings.TrimSpace(in.Content) == "" { + return webhookThoughtRequest{}, errors.New("content is required") + } + return in, nil +} + +func (h *webhookThoughtHandler) resolveWebhookProject(r *http.Request, projectName string) (*int64, error) { + projectName = strings.TrimSpace(projectName) + if projectName == "" { + return nil, nil + } + project, err := h.store.GetProject(r.Context(), projectName) + if err != nil { + return nil, err + } + return &project.NumericID, nil +} + +func buildWebhookMetadata(in webhookThoughtRequest, headerKey string, now time.Time) thoughttypes.WebhookMetadata { + sourceMetadata := in.SourceMetadata + if len(sourceMetadata) == 0 { + sourceMetadata = in.Metadata + } + return thoughttypes.WebhookMetadata{ + ReceivedAt: now.Format(time.RFC3339), + IDempotencyKey: firstNonEmpty(in.IDempotencyKey, headerKey), + ExternalID: strings.TrimSpace(in.ExternalID), + SourceMetadata: sanitizeWebhookMetadata(sourceMetadata), + } +} + +func normalizeWebhookThoughtMetadata(in webhookThoughtRequest, webhookMeta thoughttypes.WebhookMetadata, capture config.CaptureConfig) thoughttypes.ThoughtMetadata { + return metadata.Normalize(thoughttypes.ThoughtMetadata{ + People: in.People, + ActionItems: in.ActionItems, + DatesMentioned: in.DatesMentioned, + Topics: in.Topics, + Type: in.Type, + Source: firstNonEmpty(in.Source, "webhook"), + Webhook: &webhookMeta, + }, capture) +} + +func sanitizeWebhookMetadata(in map[string]any) map[string]any { + if len(in) == 0 { + return nil + } + out := make(map[string]any, len(in)) + for key, value := range in { + key = strings.TrimSpace(key) + if key == "" { + continue + } + if sanitized, ok := sanitizeWebhookMetadataValue(value, 0); ok { + out[key] = sanitized + } + } + if len(out) == 0 { + return nil + } + return out +} + +func sanitizeWebhookMetadataValue(value any, depth int) (any, bool) { + if depth > 3 { + return nil, false + } + switch v := value.(type) { + case nil, bool, float64, string: + return v, true + case []any: + if len(v) > 50 { + v = v[:50] + } + out := make([]any, 0, len(v)) + for _, item := range v { + if sanitized, ok := sanitizeWebhookMetadataValue(item, depth+1); ok { + out = append(out, sanitized) + } + } + return out, true + case map[string]any: + if len(v) > 50 { + return nil, false + } + return sanitizeWebhookMetadata(v), true + default: + return nil, false + } +} + +func writeWebhookThoughtResponse(w http.ResponseWriter, status int, out webhookThoughtResponse) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(status) + _ = json.NewEncoder(w).Encode(out) +} diff --git a/internal/app/webhooks_test.go b/internal/app/webhooks_test.go new file mode 100644 index 0000000..961b76a --- /dev/null +++ b/internal/app/webhooks_test.go @@ -0,0 +1,86 @@ +package app + +import ( + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "git.warky.dev/wdevs/amcs/internal/config" +) + +func TestParseWebhookThoughtRequestRequiresJSON(t *testing.T) { + req := httptestRequest("text/plain", `{"content":"hello"}`) + + _, err := parseWebhookThoughtRequest(req) + if err == nil { + t.Fatal("expected error for non-json content type") + } +} + +func TestParseWebhookThoughtRequestRequiresContent(t *testing.T) { + req := httptestRequest("application/json", `{"source":"n8n"}`) + + _, err := parseWebhookThoughtRequest(req) + if err == nil || !strings.Contains(err.Error(), "content is required") { + t.Fatalf("error = %v, want content required", err) + } +} + +func TestBuildWebhookMetadataUsesHeaderIdempotencyAndSanitizesMetadata(t *testing.T) { + now := time.Date(2026, 7, 15, 4, 0, 0, 0, time.UTC) + got := buildWebhookMetadata(webhookThoughtRequest{ + ExternalID: " ext-1 ", + Metadata: map[string]any{ + "service": "n8n", + "unsafe": struct{}{}, + "nested": map[string]any{"ok": true}, + }, + }, " key-1 ", now) + + if got.IDempotencyKey != "key-1" { + t.Fatalf("IDempotencyKey = %q, want key-1", got.IDempotencyKey) + } + if got.ExternalID != "ext-1" { + t.Fatalf("ExternalID = %q, want ext-1", got.ExternalID) + } + if got.ReceivedAt != "2026-07-15T04:00:00Z" { + t.Fatalf("ReceivedAt = %q", got.ReceivedAt) + } + if _, ok := got.SourceMetadata["unsafe"]; ok { + t.Fatal("unsafe metadata value was not removed") + } + if got.SourceMetadata["service"] != "n8n" { + t.Fatalf("service metadata = %#v", got.SourceMetadata["service"]) + } +} + +func TestNormalizeWebhookThoughtMetadata(t *testing.T) { + webhookMeta := buildWebhookMetadata(webhookThoughtRequest{IDempotencyKey: "abc"}, "", time.Date(2026, 7, 15, 4, 0, 0, 0, time.UTC)) + got := normalizeWebhookThoughtMetadata(webhookThoughtRequest{ + Source: "github", + Type: "task", + Topics: []string{"ci", "ci", ""}, + People: []string{" Sam "}, + }, webhookMeta, config.CaptureConfig{}) + + if got.Source != "github" { + t.Fatalf("Source = %q, want github", got.Source) + } + if got.Type != "task" { + t.Fatalf("Type = %q, want task", got.Type) + } + if len(got.Topics) != 1 || got.Topics[0] != "ci" { + t.Fatalf("Topics = %#v, want [ci]", got.Topics) + } + if got.Webhook == nil || got.Webhook.IDempotencyKey != "abc" { + t.Fatalf("Webhook = %#v, want idempotency key abc", got.Webhook) + } +} + +func httptestRequest(contentType, body string) *http.Request { + req := httptest.NewRequest(http.MethodPost, "/webhooks/thoughts", strings.NewReader(body)) + req.Header.Set("Content-Type", contentType) + return req +} diff --git a/internal/mcpserver/server.go b/internal/mcpserver/server.go index f3a127e..aca2123 100644 --- a/internal/mcpserver/server.go +++ b/internal/mcpserver/server.go @@ -19,24 +19,25 @@ const ( ) type ToolSet struct { - Version *tools.VersionTool - Capture *tools.CaptureTool - Search *tools.SearchTool - List *tools.ListTool - Stats *tools.StatsTool - Get *tools.GetTool - Update *tools.UpdateTool - Delete *tools.DeleteTool - Archive *tools.ArchiveTool - Projects *tools.ProjectsTool - Context *tools.ContextTool - Recall *tools.RecallTool - Summarize *tools.SummarizeTool - Links *tools.LinksTool - Files *tools.FilesTool - Backfill *tools.BackfillTool - Reparse *tools.ReparseMetadataTool - RetryMetadata *tools.RetryEnrichmentTool + Version *tools.VersionTool + Capture *tools.CaptureTool + Search *tools.SearchTool + List *tools.ListTool + Stats *tools.StatsTool + Get *tools.GetTool + Update *tools.UpdateTool + Delete *tools.DeleteTool + Archive *tools.ArchiveTool + DuplicateAudit *tools.DuplicateAuditTool + Projects *tools.ProjectsTool + Context *tools.ContextTool + Recall *tools.RecallTool + Summarize *tools.SummarizeTool + Links *tools.LinksTool + Files *tools.FilesTool + Backfill *tools.BackfillTool + Reparse *tools.ReparseMetadataTool + RetryMetadata *tools.RetryEnrichmentTool //Maintenance *tools.MaintenanceTool Skills *tools.SkillsTool Personas *tools.AgentPersonasTool @@ -227,6 +228,12 @@ func registerThoughtTools(server *mcp.Server, logger *slog.Logger, toolSet ToolS }, toolSet.Archive.Handle); err != nil { return err } + if err := addTool(server, logger, &mcp.Tool{ + Name: "audit_duplicates", + Description: "Dry-run duplicate audit for projects, thoughts, and metadata normalization candidates; performs no writes.", + }, toolSet.DuplicateAudit.Handle); err != nil { + return err + } if err := addTool(server, logger, &mcp.Tool{ Name: "summarize_thoughts", Description: "LLM summary of a filtered set of thoughts.", @@ -764,6 +771,7 @@ func BuildToolCatalog() []tools.ToolEntry { {Name: "update_thought", Description: "Update thought content or merge metadata.", Category: "thoughts"}, {Name: "delete_thought", Description: "Hard-delete a thought by id.", Category: "thoughts"}, {Name: "archive_thought", Description: "Archive a thought so it is hidden from default search and listing.", Category: "thoughts"}, + {Name: "audit_duplicates", Description: "Dry-run duplicate audit for exact/normalized project names, thought content, and metadata value variants. Reports candidates and recommendations; performs no writes.", Category: "admin"}, {Name: "summarize_thoughts", Description: "Produce an LLM prose summary of a filtered or searched set of thoughts.", Category: "thoughts"}, {Name: "recall_context", Description: "Recall semantically relevant and recent context for prompt injection. Combines vector similarity with recency. Falls back to full-text search when no embeddings exist.", Category: "thoughts"}, {Name: "link_thoughts", Description: "Create a typed relationship between two thoughts.", Category: "thoughts"}, diff --git a/internal/metadata/normalize.go b/internal/metadata/normalize.go index 51cd6dd..3c2c4c5 100644 --- a/internal/metadata/normalize.go +++ b/internal/metadata/normalize.go @@ -53,6 +53,7 @@ func Normalize(in thoughttypes.ThoughtMetadata, capture config.CaptureConfig) th Type: normalizeType(in.Type), Source: normalizeSource(in.Source), Attachments: normalizeAttachments(in.Attachments), + Webhook: normalizeWebhook(in.Webhook), MetadataStatus: normalizeMetadataStatus(in.MetadataStatus), MetadataUpdatedAt: strings.TrimSpace(in.MetadataUpdatedAt), MetadataLastAttemptedAt: strings.TrimSpace(in.MetadataLastAttemptedAt), @@ -201,10 +202,31 @@ func Merge(base, patch thoughttypes.ThoughtMetadata, capture config.CaptureConfi if len(patch.Attachments) > 0 { merged.Attachments = append(append([]thoughttypes.ThoughtAttachment{}, merged.Attachments...), patch.Attachments...) } + if patch.Webhook != nil { + merged.Webhook = patch.Webhook + } return Normalize(merged, capture) } +func normalizeWebhook(value *thoughttypes.WebhookMetadata) *thoughttypes.WebhookMetadata { + if value == nil { + return nil + } + out := &thoughttypes.WebhookMetadata{ + ReceivedAt: strings.TrimSpace(value.ReceivedAt), + IDempotencyKey: strings.TrimSpace(value.IDempotencyKey), + ExternalID: strings.TrimSpace(value.ExternalID), + } + if len(value.SourceMetadata) > 0 { + out.SourceMetadata = value.SourceMetadata + } + if out.ReceivedAt == "" && out.IDempotencyKey == "" && out.ExternalID == "" && len(out.SourceMetadata) == 0 { + return nil + } + return out +} + func normalizeAttachments(values []thoughttypes.ThoughtAttachment) []thoughttypes.ThoughtAttachment { seen := make(map[string]struct{}, len(values)) result := make([]thoughttypes.ThoughtAttachment, 0, len(values)) diff --git a/internal/store/thoughts.go b/internal/store/thoughts.go index fdd2f9f..0875f31 100644 --- a/internal/store/thoughts.go +++ b/internal/store/thoughts.go @@ -68,6 +68,22 @@ func (db *DB) InsertThought(ctx context.Context, thought thoughttypes.Thought, e return created, nil } +func (db *DB) GetThoughtByWebhookIDempotencyKey(ctx context.Context, key string) (thoughttypes.Thought, error) { + row := db.pool.QueryRow(ctx, ` + select id, guid, content, metadata, project_id, archived_at, created_at, updated_at + from thoughts + where metadata->'webhook'->>'idempotency_key' = $1 + order by created_at desc + limit 1 + `, strings.TrimSpace(key)) + + var model generatedmodels.ModelPublicThoughts + if err := row.Scan(&model.ID, &model.GUID, &model.Content, &model.Metadata, &model.ProjectID, &model.ArchivedAt, &model.CreatedAt, &model.UpdatedAt); err != nil { + return thoughttypes.Thought{}, err + } + return thoughtFromModel(model) +} + func (db *DB) SearchThoughts(ctx context.Context, embedding []float32, embeddingModel string, threshold float64, limit int, filter map[string]any) ([]thoughttypes.SearchResult, error) { filterJSON, err := json.Marshal(filter) if err != nil { diff --git a/internal/tools/duplicate_audit.go b/internal/tools/duplicate_audit.go new file mode 100644 index 0000000..f503fca --- /dev/null +++ b/internal/tools/duplicate_audit.go @@ -0,0 +1,305 @@ +package tools + +import ( + "context" + "regexp" + "sort" + "strings" + + "github.com/google/uuid" + "github.com/modelcontextprotocol/go-sdk/mcp" + + "git.warky.dev/wdevs/amcs/internal/config" + "git.warky.dev/wdevs/amcs/internal/session" + "git.warky.dev/wdevs/amcs/internal/store" + thoughttypes "git.warky.dev/wdevs/amcs/internal/types" +) + +const defaultDuplicateAuditLimit = 50 + +var duplicateWhitespace = regexp.MustCompile(`\s+`) + +type DuplicateAuditTool struct { + store *store.DB + search config.SearchConfig + sessions *session.ActiveProjects +} + +type DuplicateAuditInput struct { + Project string `json:"project,omitempty" jsonschema:"optional project name or id to scope thought duplicate scanning"` + Limit int `json:"limit,omitempty" jsonschema:"maximum candidate groups to return"` + ThoughtLimit int `json:"thought_limit,omitempty" jsonschema:"maximum thoughts to scan for duplicate content; defaults to search limit"` + IncludeArchived bool `json:"include_archived,omitempty" jsonschema:"include archived thoughts in the scan"` +} + +type DuplicateAuditOutput struct { + Report DuplicateAuditReport `json:"report"` +} + +type DuplicateAuditReport struct { + Summary DuplicateAuditSummary `json:"summary"` + ProjectCandidates []DuplicateProjectCandidate `json:"project_candidates"` + ThoughtCandidates []DuplicateThoughtCandidate `json:"thought_candidates"` + MetadataCandidates []MetadataCleanupCandidate `json:"metadata_candidates"` + RecommendedNextStep string `json:"recommended_next_step"` +} + +type DuplicateAuditSummary struct { + DryRun bool `json:"dry_run"` + ScannedProjects int `json:"scanned_projects"` + ScannedThoughts int `json:"scanned_thoughts"` + ProjectCandidateGroups int `json:"project_candidate_groups"` + ThoughtCandidateGroups int `json:"thought_candidate_groups"` + MetadataCandidateGroups int `json:"metadata_candidate_groups"` + Truncated bool `json:"truncated"` +} + +type DuplicateProjectCandidate struct { + MatchType string `json:"match_type"` + NormalizedValue string `json:"normalized_value"` + Confidence float64 `json:"confidence"` + RecommendedCanonicalID uuid.UUID `json:"recommended_canonical_id"` + Projects []thoughttypes.ProjectSummary `json:"projects"` +} + +type DuplicateThoughtCandidate struct { + MatchType string `json:"match_type"` + NormalizedValue string `json:"normalized_value"` + Confidence float64 `json:"confidence"` + RecommendedCanonicalID uuid.UUID `json:"recommended_canonical_id"` + Thoughts []thoughttypes.Thought `json:"thoughts"` +} + +type MetadataCleanupCandidate struct { + Field string `json:"field"` + NormalizedValue string `json:"normalized_value"` + Values []string `json:"values"` + Occurrences int `json:"occurrences"` + RecommendedValue string `json:"recommended_value"` + RecommendedAction string `json:"recommended_action"` +} + +func NewDuplicateAuditTool(db *store.DB, search config.SearchConfig, sessions *session.ActiveProjects) *DuplicateAuditTool { + return &DuplicateAuditTool{store: db, search: search, sessions: sessions} +} + +func (t *DuplicateAuditTool) Handle(ctx context.Context, req *mcp.CallToolRequest, in DuplicateAuditInput) (*mcp.CallToolResult, DuplicateAuditOutput, error) { + project, err := resolveProject(ctx, t.store, t.sessions, req, in.Project, false) + if err != nil { + return nil, DuplicateAuditOutput{}, err + } + var projectID *int64 + if project != nil { + projectID = &project.NumericID + } + + projects, err := t.store.ListProjects(ctx) + if err != nil { + return nil, DuplicateAuditOutput{}, err + } + + thoughtLimit := normalizeLimit(in.ThoughtLimit, t.search) + if in.ThoughtLimit <= 0 { + thoughtLimit = normalizeLimit(0, t.search) + } + thoughts, err := t.store.ListThoughts(ctx, thoughttypes.ListFilter{Limit: thoughtLimit, ProjectID: projectID, IncludeArchived: in.IncludeArchived}) + if err != nil { + return nil, DuplicateAuditOutput{}, err + } + + if project != nil { + _ = t.store.TouchProject(ctx, project.NumericID) + } + return nil, DuplicateAuditOutput{Report: buildDuplicateAuditReport(projects, thoughts, in)}, nil +} + +func buildDuplicateAuditReport(projects []thoughttypes.ProjectSummary, thoughts []thoughttypes.Thought, in DuplicateAuditInput) DuplicateAuditReport { + limit := in.Limit + if limit <= 0 { + limit = defaultDuplicateAuditLimit + } + + projectCandidates, projectTruncated := duplicateProjectCandidates(projects, limit) + thoughtCandidates, thoughtTruncated := duplicateThoughtCandidates(thoughts, limit) + metadataCandidates, metadataTruncated := metadataCleanupCandidates(thoughts, limit) + + return DuplicateAuditReport{ + Summary: DuplicateAuditSummary{ + DryRun: true, + ScannedProjects: len(projects), + ScannedThoughts: len(thoughts), + ProjectCandidateGroups: len(projectCandidates), + ThoughtCandidateGroups: len(thoughtCandidates), + MetadataCandidateGroups: len(metadataCandidates), + Truncated: projectTruncated || thoughtTruncated || metadataTruncated, + }, + ProjectCandidates: projectCandidates, + ThoughtCandidates: thoughtCandidates, + MetadataCandidates: metadataCandidates, + RecommendedNextStep: "Review candidates manually, then use explicit archive/update/merge tools; this audit performs no writes.", + } +} + +func duplicateProjectCandidates(projects []thoughttypes.ProjectSummary, limit int) ([]DuplicateProjectCandidate, bool) { + groups := map[string][]thoughttypes.ProjectSummary{} + for _, project := range projects { + key := normalizeDuplicateText(project.Name) + if key != "" { + groups[key] = append(groups[key], project) + } + } + keys := sortedDuplicateKeys(groups) + out := make([]DuplicateProjectCandidate, 0, min(len(keys), limit)) + for _, key := range keys { + items := groups[key] + sort.SliceStable(items, func(i, j int) bool { + if items[i].ThoughtCount != items[j].ThoughtCount { + return items[i].ThoughtCount > items[j].ThoughtCount + } + return items[i].CreatedAt.Before(items[j].CreatedAt) + }) + matchType := "normalized_name" + if allProjectNamesExact(items) { + matchType = "exact_name" + } + out = append(out, DuplicateProjectCandidate{MatchType: matchType, NormalizedValue: key, Confidence: 1.0, RecommendedCanonicalID: items[0].ID, Projects: items}) + if len(out) == limit { + return out, len(keys) > limit + } + } + return out, false +} + +func duplicateThoughtCandidates(thoughts []thoughttypes.Thought, limit int) ([]DuplicateThoughtCandidate, bool) { + groups := map[string][]thoughttypes.Thought{} + for _, thought := range thoughts { + key := normalizeDuplicateText(thought.Content) + if key != "" { + groups[key] = append(groups[key], thought) + } + } + keys := sortedDuplicateKeys(groups) + out := make([]DuplicateThoughtCandidate, 0, min(len(keys), limit)) + for _, key := range keys { + items := groups[key] + sort.SliceStable(items, func(i, j int) bool { return items[i].CreatedAt.Before(items[j].CreatedAt) }) + matchType := "normalized_content" + if allThoughtContentExact(items) { + matchType = "exact_content" + } + out = append(out, DuplicateThoughtCandidate{MatchType: matchType, NormalizedValue: key, Confidence: 1.0, RecommendedCanonicalID: items[0].GUID, Thoughts: items}) + if len(out) == limit { + return out, len(keys) > limit + } + } + return out, false +} + +func metadataCleanupCandidates(thoughts []thoughttypes.Thought, limit int) ([]MetadataCleanupCandidate, bool) { + groups := map[string]map[string]int{} + add := func(field, value string) { + trimmed := strings.TrimSpace(value) + key := normalizeDuplicateText(trimmed) + if key == "" || trimmed == "" { + return + } + bucketKey := field + "\x00" + key + if groups[bucketKey] == nil { + groups[bucketKey] = map[string]int{} + } + groups[bucketKey][trimmed]++ + } + for _, thought := range thoughts { + add("type", thought.Metadata.Type) + add("source", thought.Metadata.Source) + for _, topic := range thought.Metadata.Topics { + add("topics", topic) + } + for _, person := range thought.Metadata.People { + add("people", person) + } + } + + keys := make([]string, 0, len(groups)) + for key, values := range groups { + if len(values) > 1 { + keys = append(keys, key) + } + } + sort.Strings(keys) + + out := make([]MetadataCleanupCandidate, 0, min(len(keys), limit)) + for _, key := range keys { + parts := strings.SplitN(key, "\x00", 2) + values, occurrences, recommended := valuesByCount(groups[key]) + out = append(out, MetadataCleanupCandidate{ + Field: parts[0], + NormalizedValue: parts[1], + Values: values, + Occurrences: occurrences, + RecommendedValue: recommended, + RecommendedAction: "preview bulk metadata remap before applying; no changes made by audit", + }) + if len(out) == limit { + return out, len(keys) > limit + } + } + return out, false +} + +func sortedDuplicateKeys[T any](groups map[string][]T) []string { + keys := make([]string, 0, len(groups)) + for key, items := range groups { + if len(items) > 1 { + keys = append(keys, key) + } + } + sort.Strings(keys) + return keys +} + +func valuesByCount(counts map[string]int) ([]string, int, string) { + values := make([]string, 0, len(counts)) + occurrences := 0 + for value, count := range counts { + values = append(values, value) + occurrences += count + } + sort.Slice(values, func(i, j int) bool { + if counts[values[i]] != counts[values[j]] { + return counts[values[i]] > counts[values[j]] + } + return values[i] < values[j] + }) + return values, occurrences, values[0] +} + +func normalizeDuplicateText(value string) string { + return duplicateWhitespace.ReplaceAllString(strings.ToLower(strings.TrimSpace(value)), " ") +} + +func allProjectNamesExact(projects []thoughttypes.ProjectSummary) bool { + if len(projects) == 0 { + return false + } + first := strings.TrimSpace(projects[0].Name) + for _, project := range projects[1:] { + if strings.TrimSpace(project.Name) != first { + return false + } + } + return true +} + +func allThoughtContentExact(thoughts []thoughttypes.Thought) bool { + if len(thoughts) == 0 { + return false + } + first := strings.TrimSpace(thoughts[0].Content) + for _, thought := range thoughts[1:] { + if strings.TrimSpace(thought.Content) != first { + return false + } + } + return true +} diff --git a/internal/tools/duplicate_audit_test.go b/internal/tools/duplicate_audit_test.go new file mode 100644 index 0000000..93455d3 --- /dev/null +++ b/internal/tools/duplicate_audit_test.go @@ -0,0 +1,76 @@ +package tools + +import ( + "testing" + "time" + + "github.com/google/uuid" + + thoughttypes "git.warky.dev/wdevs/amcs/internal/types" +) + +func TestBuildDuplicateAuditGroupsProjectsAndThoughtsSafely(t *testing.T) { + now := time.Date(2026, 4, 12, 20, 5, 0, 0, time.UTC) + projects := []thoughttypes.ProjectSummary{ + {Project: thoughttypes.Project{ID: uuid.MustParse("11111111-1111-1111-1111-111111111111"), NumericID: 1, Name: "AMCS", CreatedAt: now}, ThoughtCount: 3}, + {Project: thoughttypes.Project{ID: uuid.MustParse("22222222-2222-2222-2222-222222222222"), NumericID: 2, Name: "amcs ", CreatedAt: now.Add(time.Hour)}, ThoughtCount: 1}, + {Project: thoughttypes.Project{ID: uuid.MustParse("33333333-3333-3333-3333-333333333333"), NumericID: 3, Name: "other", CreatedAt: now}, ThoughtCount: 1}, + } + thoughts := []thoughttypes.Thought{ + {ID: 10, GUID: uuid.MustParse("aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa"), Content: "Same text", Metadata: thoughttypes.ThoughtMetadata{Type: "Note", Topics: []string{"Go", "go"}, People: []string{"Alice"}}, CreatedAt: now}, + {ID: 11, GUID: uuid.MustParse("bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb"), Content: " same text ", Metadata: thoughttypes.ThoughtMetadata{Type: "note", Topics: []string{"go"}, People: []string{"alice"}}, CreatedAt: now.Add(time.Hour)}, + {ID: 12, GUID: uuid.MustParse("cccccccc-cccc-cccc-cccc-cccccccccccc"), Content: "unique", Metadata: thoughttypes.ThoughtMetadata{Type: "Task", Topics: []string{"Tasks"}}, CreatedAt: now}, + } + + report := buildDuplicateAuditReport(projects, thoughts, DuplicateAuditInput{}) + + if len(report.ProjectCandidates) != 1 { + t.Fatalf("project candidates = %d, want 1: %#v", len(report.ProjectCandidates), report.ProjectCandidates) + } + projectCandidate := report.ProjectCandidates[0] + if projectCandidate.MatchType != "normalized_name" || projectCandidate.NormalizedValue != "amcs" || len(projectCandidate.Projects) != 2 { + t.Fatalf("project candidate mismatch: %#v", projectCandidate) + } + if projectCandidate.RecommendedCanonicalID != projects[0].ID { + t.Fatalf("recommended project = %s, want %s", projectCandidate.RecommendedCanonicalID, projects[0].ID) + } + + if len(report.ThoughtCandidates) != 1 { + t.Fatalf("thought candidates = %d, want 1: %#v", len(report.ThoughtCandidates), report.ThoughtCandidates) + } + thoughtCandidate := report.ThoughtCandidates[0] + if thoughtCandidate.MatchType != "normalized_content" || thoughtCandidate.NormalizedValue != "same text" || len(thoughtCandidate.Thoughts) != 2 { + t.Fatalf("thought candidate mismatch: %#v", thoughtCandidate) + } + if thoughtCandidate.RecommendedCanonicalID != thoughts[0].GUID { + t.Fatalf("recommended thought = %s, want %s", thoughtCandidate.RecommendedCanonicalID, thoughts[0].GUID) + } + + if len(report.MetadataCandidates) != 3 { + t.Fatalf("metadata candidates = %d, want 3: %#v", len(report.MetadataCandidates), report.MetadataCandidates) + } + if report.Summary.ProjectCandidateGroups != 1 || report.Summary.ThoughtCandidateGroups != 1 || report.Summary.MetadataCandidateGroups != 3 { + t.Fatalf("summary mismatch: %#v", report.Summary) + } + if report.Summary.DryRun != true { + t.Fatalf("dry run = false, want true") + } +} + +func TestBuildDuplicateAuditRespectsLimit(t *testing.T) { + projects := []thoughttypes.ProjectSummary{ + {Project: thoughttypes.Project{ID: uuid.MustParse("11111111-1111-1111-1111-111111111111"), Name: "Alpha"}}, + {Project: thoughttypes.Project{ID: uuid.MustParse("22222222-2222-2222-2222-222222222222"), Name: "alpha"}}, + {Project: thoughttypes.Project{ID: uuid.MustParse("33333333-3333-3333-3333-333333333333"), Name: "Beta"}}, + {Project: thoughttypes.Project{ID: uuid.MustParse("44444444-4444-4444-4444-444444444444"), Name: "beta"}}, + } + + report := buildDuplicateAuditReport(projects, nil, DuplicateAuditInput{Limit: 1}) + + if len(report.ProjectCandidates) != 1 { + t.Fatalf("project candidates = %d, want 1", len(report.ProjectCandidates)) + } + if report.Summary.Truncated != true { + t.Fatalf("truncated = false, want true") + } +} diff --git a/internal/types/thought.go b/internal/types/thought.go index 4f01af9..0b278af 100644 --- a/internal/types/thought.go +++ b/internal/types/thought.go @@ -14,12 +14,20 @@ type ThoughtMetadata struct { Type string `json:"type"` Source string `json:"source"` Attachments []ThoughtAttachment `json:"attachments,omitempty"` + Webhook *WebhookMetadata `json:"webhook,omitempty"` MetadataStatus string `json:"metadata_status,omitempty"` MetadataUpdatedAt string `json:"metadata_updated_at,omitempty"` MetadataLastAttemptedAt string `json:"metadata_last_attempted_at,omitempty"` MetadataError string `json:"metadata_error,omitempty"` } +type WebhookMetadata struct { + ReceivedAt string `json:"received_at"` + IDempotencyKey string `json:"idempotency_key,omitempty"` + ExternalID string `json:"external_id,omitempty"` + SourceMetadata map[string]any `json:"source_metadata,omitempty"` +} + type ThoughtAttachment struct { FileID uuid.UUID `json:"file_id"` Name string `json:"name"` @@ -30,19 +38,19 @@ type ThoughtAttachment struct { } type StoredFile struct { - ID int64 `json:"id"` - GUID uuid.UUID `json:"guid"` - ThoughtID *int64 `json:"thought_id,omitempty"` - ProjectID *int64 `json:"project_id,omitempty"` - Name string `json:"name"` - MediaType string `json:"media_type"` - Kind string `json:"kind"` - Encoding string `json:"encoding"` - SizeBytes int64 `json:"size_bytes"` - SHA256 string `json:"sha256"` - Content []byte `json:"-"` - CreatedAt time.Time `json:"created_at"` - UpdatedAt time.Time `json:"updated_at"` + ID int64 `json:"id"` + GUID uuid.UUID `json:"guid"` + ThoughtID *int64 `json:"thought_id,omitempty"` + ProjectID *int64 `json:"project_id,omitempty"` + Name string `json:"name"` + MediaType string `json:"media_type"` + Kind string `json:"kind"` + Encoding string `json:"encoding"` + SizeBytes int64 `json:"size_bytes"` + SHA256 string `json:"sha256"` + Content []byte `json:"-"` + CreatedAt time.Time `json:"created_at"` + UpdatedAt time.Time `json:"updated_at"` } type StoredFileFilter struct {