Improve thought enrichment reliability

internal/tools/metadata_retry.go

@@ -28,12 +28,42 @@ type MetadataRetryer struct {
 	sessions        *session.ActiveProjects
 	metadataTimeout time.Duration
 	logger          *slog.Logger
+	lock            *RetryLocker
 }
 
 type RetryMetadataTool struct {
 	retryer *MetadataRetryer
 }
 
+type RetryLocker struct {
+	mu    sync.Mutex
+	locks map[uuid.UUID]time.Time
+}
+
+func NewRetryLocker() *RetryLocker {
+	return &RetryLocker{locks: map[uuid.UUID]time.Time{}}
+}
+
+func (l *RetryLocker) Acquire(id uuid.UUID, ttl time.Duration) bool {
+	l.mu.Lock()
+	defer l.mu.Unlock()
+	if l.locks == nil {
+		l.locks = map[uuid.UUID]time.Time{}
+	}
+	now := time.Now()
+	if exp, ok := l.locks[id]; ok && exp.After(now) {
+		return false
+	}
+	l.locks[id] = now.Add(ttl)
+	return true
+}
+
+func (l *RetryLocker) Release(id uuid.UUID) {
+	l.mu.Lock()
+	defer l.mu.Unlock()
+	delete(l.locks, id)
+}
+
 type RetryMetadataInput struct {
 	Project         string `json:"project,omitempty" jsonschema:"optional project name or id to scope the retry"`
 	Limit           int    `json:"limit,omitempty" jsonschema:"maximum number of thoughts to process in one call; defaults to 100"`
@@ -69,6 +99,7 @@ func NewMetadataRetryer(backgroundCtx context.Context, db *store.DB, provider ai
 		sessions:        sessions,
 		metadataTimeout: metadataTimeout,
 		logger:          logger,
+		lock:            NewRetryLocker(),
 	}
 }
 
@@ -82,6 +113,10 @@ func (t *RetryMetadataTool) Handle(ctx context.Context, req *mcp.CallToolRequest
 
 func (r *MetadataRetryer) QueueThought(id uuid.UUID) {
 	go func() {
+		if !r.lock.Acquire(id, 15*time.Minute) {
+			return
+		}
+		defer r.lock.Release(id)
 		if _, err := r.retryOne(r.backgroundCtx, id); err != nil {
 			r.logger.Warn("background metadata retry failed", slog.String("thought_id", id.String()), slog.String("error", err.Error()))
 		}
@@ -138,7 +173,14 @@ func (r *MetadataRetryer) Handle(ctx context.Context, req *mcp.CallToolRequest,
 			out.Retried++
 			mu.Unlock()
 
+			if !r.lock.Acquire(thought.ID, 15*time.Minute) {
+				mu.Lock()
+				out.Skipped++
+				mu.Unlock()
+				return
+			}
 			updated, err := r.retryOne(ctx, thought.ID)
+			r.lock.Release(thought.ID)
 			if err != nil {
 				mu.Lock()
 				out.Failures = append(out.Failures, RetryMetadataFailure{ID: thought.ID.String(), Error: err.Error()})