306 lines
10 KiB
Go
306 lines
10 KiB
Go
package tools
|
|
|
|
import (
|
|
"context"
|
|
"regexp"
|
|
"sort"
|
|
"strings"
|
|
|
|
"github.com/google/uuid"
|
|
"github.com/modelcontextprotocol/go-sdk/mcp"
|
|
|
|
"git.warky.dev/wdevs/amcs/internal/config"
|
|
"git.warky.dev/wdevs/amcs/internal/session"
|
|
"git.warky.dev/wdevs/amcs/internal/store"
|
|
thoughttypes "git.warky.dev/wdevs/amcs/internal/types"
|
|
)
|
|
|
|
const defaultDuplicateAuditLimit = 50
|
|
|
|
var duplicateWhitespace = regexp.MustCompile(`\s+`)
|
|
|
|
type DuplicateAuditTool struct {
|
|
store *store.DB
|
|
search config.SearchConfig
|
|
sessions *session.ActiveProjects
|
|
}
|
|
|
|
type DuplicateAuditInput struct {
|
|
Project string `json:"project,omitempty" jsonschema:"optional project name or id to scope thought duplicate scanning"`
|
|
Limit int `json:"limit,omitempty" jsonschema:"maximum candidate groups to return"`
|
|
ThoughtLimit int `json:"thought_limit,omitempty" jsonschema:"maximum thoughts to scan for duplicate content; defaults to search limit"`
|
|
IncludeArchived bool `json:"include_archived,omitempty" jsonschema:"include archived thoughts in the scan"`
|
|
}
|
|
|
|
type DuplicateAuditOutput struct {
|
|
Report DuplicateAuditReport `json:"report"`
|
|
}
|
|
|
|
type DuplicateAuditReport struct {
|
|
Summary DuplicateAuditSummary `json:"summary"`
|
|
ProjectCandidates []DuplicateProjectCandidate `json:"project_candidates"`
|
|
ThoughtCandidates []DuplicateThoughtCandidate `json:"thought_candidates"`
|
|
MetadataCandidates []MetadataCleanupCandidate `json:"metadata_candidates"`
|
|
RecommendedNextStep string `json:"recommended_next_step"`
|
|
}
|
|
|
|
type DuplicateAuditSummary struct {
|
|
DryRun bool `json:"dry_run"`
|
|
ScannedProjects int `json:"scanned_projects"`
|
|
ScannedThoughts int `json:"scanned_thoughts"`
|
|
ProjectCandidateGroups int `json:"project_candidate_groups"`
|
|
ThoughtCandidateGroups int `json:"thought_candidate_groups"`
|
|
MetadataCandidateGroups int `json:"metadata_candidate_groups"`
|
|
Truncated bool `json:"truncated"`
|
|
}
|
|
|
|
type DuplicateProjectCandidate struct {
|
|
MatchType string `json:"match_type"`
|
|
NormalizedValue string `json:"normalized_value"`
|
|
Confidence float64 `json:"confidence"`
|
|
RecommendedCanonicalID uuid.UUID `json:"recommended_canonical_id"`
|
|
Projects []thoughttypes.ProjectSummary `json:"projects"`
|
|
}
|
|
|
|
type DuplicateThoughtCandidate struct {
|
|
MatchType string `json:"match_type"`
|
|
NormalizedValue string `json:"normalized_value"`
|
|
Confidence float64 `json:"confidence"`
|
|
RecommendedCanonicalID uuid.UUID `json:"recommended_canonical_id"`
|
|
Thoughts []thoughttypes.Thought `json:"thoughts"`
|
|
}
|
|
|
|
type MetadataCleanupCandidate struct {
|
|
Field string `json:"field"`
|
|
NormalizedValue string `json:"normalized_value"`
|
|
Values []string `json:"values"`
|
|
Occurrences int `json:"occurrences"`
|
|
RecommendedValue string `json:"recommended_value"`
|
|
RecommendedAction string `json:"recommended_action"`
|
|
}
|
|
|
|
func NewDuplicateAuditTool(db *store.DB, search config.SearchConfig, sessions *session.ActiveProjects) *DuplicateAuditTool {
|
|
return &DuplicateAuditTool{store: db, search: search, sessions: sessions}
|
|
}
|
|
|
|
func (t *DuplicateAuditTool) Handle(ctx context.Context, req *mcp.CallToolRequest, in DuplicateAuditInput) (*mcp.CallToolResult, DuplicateAuditOutput, error) {
|
|
project, err := resolveProject(ctx, t.store, t.sessions, req, in.Project, false)
|
|
if err != nil {
|
|
return nil, DuplicateAuditOutput{}, err
|
|
}
|
|
var projectID *int64
|
|
if project != nil {
|
|
projectID = &project.NumericID
|
|
}
|
|
|
|
projects, err := t.store.ListProjects(ctx)
|
|
if err != nil {
|
|
return nil, DuplicateAuditOutput{}, err
|
|
}
|
|
|
|
thoughtLimit := normalizeLimit(in.ThoughtLimit, t.search)
|
|
if in.ThoughtLimit <= 0 {
|
|
thoughtLimit = normalizeLimit(0, t.search)
|
|
}
|
|
thoughts, err := t.store.ListThoughts(ctx, thoughttypes.ListFilter{Limit: thoughtLimit, ProjectID: projectID, IncludeArchived: in.IncludeArchived})
|
|
if err != nil {
|
|
return nil, DuplicateAuditOutput{}, err
|
|
}
|
|
|
|
if project != nil {
|
|
_ = t.store.TouchProject(ctx, project.NumericID)
|
|
}
|
|
return nil, DuplicateAuditOutput{Report: buildDuplicateAuditReport(projects, thoughts, in)}, nil
|
|
}
|
|
|
|
func buildDuplicateAuditReport(projects []thoughttypes.ProjectSummary, thoughts []thoughttypes.Thought, in DuplicateAuditInput) DuplicateAuditReport {
|
|
limit := in.Limit
|
|
if limit <= 0 {
|
|
limit = defaultDuplicateAuditLimit
|
|
}
|
|
|
|
projectCandidates, projectTruncated := duplicateProjectCandidates(projects, limit)
|
|
thoughtCandidates, thoughtTruncated := duplicateThoughtCandidates(thoughts, limit)
|
|
metadataCandidates, metadataTruncated := metadataCleanupCandidates(thoughts, limit)
|
|
|
|
return DuplicateAuditReport{
|
|
Summary: DuplicateAuditSummary{
|
|
DryRun: true,
|
|
ScannedProjects: len(projects),
|
|
ScannedThoughts: len(thoughts),
|
|
ProjectCandidateGroups: len(projectCandidates),
|
|
ThoughtCandidateGroups: len(thoughtCandidates),
|
|
MetadataCandidateGroups: len(metadataCandidates),
|
|
Truncated: projectTruncated || thoughtTruncated || metadataTruncated,
|
|
},
|
|
ProjectCandidates: projectCandidates,
|
|
ThoughtCandidates: thoughtCandidates,
|
|
MetadataCandidates: metadataCandidates,
|
|
RecommendedNextStep: "Review candidates manually, then use explicit archive/update/merge tools; this audit performs no writes.",
|
|
}
|
|
}
|
|
|
|
func duplicateProjectCandidates(projects []thoughttypes.ProjectSummary, limit int) ([]DuplicateProjectCandidate, bool) {
|
|
groups := map[string][]thoughttypes.ProjectSummary{}
|
|
for _, project := range projects {
|
|
key := normalizeDuplicateText(project.Name)
|
|
if key != "" {
|
|
groups[key] = append(groups[key], project)
|
|
}
|
|
}
|
|
keys := sortedDuplicateKeys(groups)
|
|
out := make([]DuplicateProjectCandidate, 0, min(len(keys), limit))
|
|
for _, key := range keys {
|
|
items := groups[key]
|
|
sort.SliceStable(items, func(i, j int) bool {
|
|
if items[i].ThoughtCount != items[j].ThoughtCount {
|
|
return items[i].ThoughtCount > items[j].ThoughtCount
|
|
}
|
|
return items[i].CreatedAt.Before(items[j].CreatedAt)
|
|
})
|
|
matchType := "normalized_name"
|
|
if allProjectNamesExact(items) {
|
|
matchType = "exact_name"
|
|
}
|
|
out = append(out, DuplicateProjectCandidate{MatchType: matchType, NormalizedValue: key, Confidence: 1.0, RecommendedCanonicalID: items[0].ID, Projects: items})
|
|
if len(out) == limit {
|
|
return out, len(keys) > limit
|
|
}
|
|
}
|
|
return out, false
|
|
}
|
|
|
|
func duplicateThoughtCandidates(thoughts []thoughttypes.Thought, limit int) ([]DuplicateThoughtCandidate, bool) {
|
|
groups := map[string][]thoughttypes.Thought{}
|
|
for _, thought := range thoughts {
|
|
key := normalizeDuplicateText(thought.Content)
|
|
if key != "" {
|
|
groups[key] = append(groups[key], thought)
|
|
}
|
|
}
|
|
keys := sortedDuplicateKeys(groups)
|
|
out := make([]DuplicateThoughtCandidate, 0, min(len(keys), limit))
|
|
for _, key := range keys {
|
|
items := groups[key]
|
|
sort.SliceStable(items, func(i, j int) bool { return items[i].CreatedAt.Before(items[j].CreatedAt) })
|
|
matchType := "normalized_content"
|
|
if allThoughtContentExact(items) {
|
|
matchType = "exact_content"
|
|
}
|
|
out = append(out, DuplicateThoughtCandidate{MatchType: matchType, NormalizedValue: key, Confidence: 1.0, RecommendedCanonicalID: items[0].GUID, Thoughts: items})
|
|
if len(out) == limit {
|
|
return out, len(keys) > limit
|
|
}
|
|
}
|
|
return out, false
|
|
}
|
|
|
|
func metadataCleanupCandidates(thoughts []thoughttypes.Thought, limit int) ([]MetadataCleanupCandidate, bool) {
|
|
groups := map[string]map[string]int{}
|
|
add := func(field, value string) {
|
|
trimmed := strings.TrimSpace(value)
|
|
key := normalizeDuplicateText(trimmed)
|
|
if key == "" || trimmed == "" {
|
|
return
|
|
}
|
|
bucketKey := field + "\x00" + key
|
|
if groups[bucketKey] == nil {
|
|
groups[bucketKey] = map[string]int{}
|
|
}
|
|
groups[bucketKey][trimmed]++
|
|
}
|
|
for _, thought := range thoughts {
|
|
add("type", thought.Metadata.Type)
|
|
add("source", thought.Metadata.Source)
|
|
for _, topic := range thought.Metadata.Topics {
|
|
add("topics", topic)
|
|
}
|
|
for _, person := range thought.Metadata.People {
|
|
add("people", person)
|
|
}
|
|
}
|
|
|
|
keys := make([]string, 0, len(groups))
|
|
for key, values := range groups {
|
|
if len(values) > 1 {
|
|
keys = append(keys, key)
|
|
}
|
|
}
|
|
sort.Strings(keys)
|
|
|
|
out := make([]MetadataCleanupCandidate, 0, min(len(keys), limit))
|
|
for _, key := range keys {
|
|
parts := strings.SplitN(key, "\x00", 2)
|
|
values, occurrences, recommended := valuesByCount(groups[key])
|
|
out = append(out, MetadataCleanupCandidate{
|
|
Field: parts[0],
|
|
NormalizedValue: parts[1],
|
|
Values: values,
|
|
Occurrences: occurrences,
|
|
RecommendedValue: recommended,
|
|
RecommendedAction: "preview bulk metadata remap before applying; no changes made by audit",
|
|
})
|
|
if len(out) == limit {
|
|
return out, len(keys) > limit
|
|
}
|
|
}
|
|
return out, false
|
|
}
|
|
|
|
func sortedDuplicateKeys[T any](groups map[string][]T) []string {
|
|
keys := make([]string, 0, len(groups))
|
|
for key, items := range groups {
|
|
if len(items) > 1 {
|
|
keys = append(keys, key)
|
|
}
|
|
}
|
|
sort.Strings(keys)
|
|
return keys
|
|
}
|
|
|
|
func valuesByCount(counts map[string]int) ([]string, int, string) {
|
|
values := make([]string, 0, len(counts))
|
|
occurrences := 0
|
|
for value, count := range counts {
|
|
values = append(values, value)
|
|
occurrences += count
|
|
}
|
|
sort.Slice(values, func(i, j int) bool {
|
|
if counts[values[i]] != counts[values[j]] {
|
|
return counts[values[i]] > counts[values[j]]
|
|
}
|
|
return values[i] < values[j]
|
|
})
|
|
return values, occurrences, values[0]
|
|
}
|
|
|
|
func normalizeDuplicateText(value string) string {
|
|
return duplicateWhitespace.ReplaceAllString(strings.ToLower(strings.TrimSpace(value)), " ")
|
|
}
|
|
|
|
func allProjectNamesExact(projects []thoughttypes.ProjectSummary) bool {
|
|
if len(projects) == 0 {
|
|
return false
|
|
}
|
|
first := strings.TrimSpace(projects[0].Name)
|
|
for _, project := range projects[1:] {
|
|
if strings.TrimSpace(project.Name) != first {
|
|
return false
|
|
}
|
|
}
|
|
return true
|
|
}
|
|
|
|
func allThoughtContentExact(thoughts []thoughttypes.Thought) bool {
|
|
if len(thoughts) == 0 {
|
|
return false
|
|
}
|
|
first := strings.TrimSpace(thoughts[0].Content)
|
|
for _, thought := range thoughts[1:] {
|
|
if strings.TrimSpace(thought.Content) != first {
|
|
return false
|
|
}
|
|
}
|
|
return true
|
|
}
|