wdevs/amcs
5
0
Fork 0
Code Issues 10 Pull Requests Actions Packages Projects Releases 1 Wiki Activity

feat: implement file upload handler and related functionality

Browse Source 
- Added file upload handler to process both multipart and raw file uploads.
- Implemented parsing logic for upload requests, including handling file metadata.
- Introduced SaveFileDecodedInput structure for handling decoded file uploads.
- Created unit tests for file upload parsing and validation.

feat: add metadata retry configuration and functionality

- Introduced MetadataRetryConfig to the application configuration.
- Implemented MetadataRetryer to handle retrying metadata extraction for thoughts.
- Added new tool for retrying failed metadata extractions.
- Updated thought metadata structure to include status and timestamps for metadata processing.

fix: enhance metadata normalization and error handling

- Updated metadata normalization functions to track status and errors.
- Improved handling of metadata extraction failures during thought updates and captures.
- Ensured that metadata status is correctly set during various operations.

refactor: streamline file saving logic in FilesTool

- Refactored Save method in FilesTool to utilize new SaveDecoded method.
- Simplified project and thought ID resolution logic during file saving.
This commit is contained in:
Warky
2026-03-30 22:57:21 +02:00
parent 7f2b2b9fee
commit 72b4f7ce3d
21 changed files with 890 additions and 126 deletions
Download Patch File Download Diff File Expand all files Collapse all files

80
internal/metadata/normalize.go
View File

@@ -3,15 +3,19 @@ package metadata
import (
"sort"
"strings"
"time"
"git.warky.dev/wdevs/amcs/internal/config"
thoughttypes "git.warky.dev/wdevs/amcs/internal/types"
)
const (
DefaultType = "observation"
DefaultTopicFallback = "uncategorized"
maxTopics = 10
DefaultType = "observation"
DefaultTopicFallback = "uncategorized"
MetadataStatusComplete = "complete"
MetadataStatusPending = "pending"
MetadataStatusFailed = "failed"
maxTopics = 10
)
var allowedTypes = map[string]struct{}{
@@ -36,18 +40,23 @@ func Fallback(capture config.CaptureConfig) thoughttypes.ThoughtMetadata {
Type: normalizeType(capture.MetadataDefaults.Type),
Source: normalizeSource(capture.Source),
Attachments: []thoughttypes.ThoughtAttachment{},
MetadataStatus: MetadataStatusComplete,
}
}
func Normalize(in thoughttypes.ThoughtMetadata, capture config.CaptureConfig) thoughttypes.ThoughtMetadata {
out := thoughttypes.ThoughtMetadata{
People: normalizeList(in.People, 0),
ActionItems: normalizeList(in.ActionItems, 0),
DatesMentioned: normalizeList(in.DatesMentioned, 0),
Topics: normalizeList(in.Topics, maxTopics),
Type: normalizeType(in.Type),
Source: normalizeSource(in.Source),
Attachments: normalizeAttachments(in.Attachments),
People: normalizeList(in.People, 0),
ActionItems: normalizeList(in.ActionItems, 0),
DatesMentioned: normalizeList(in.DatesMentioned, 0),
Topics: normalizeList(in.Topics, maxTopics),
Type: normalizeType(in.Type),
Source: normalizeSource(in.Source),
Attachments: normalizeAttachments(in.Attachments),
MetadataStatus: normalizeMetadataStatus(in.MetadataStatus),
MetadataUpdatedAt: strings.TrimSpace(in.MetadataUpdatedAt),
MetadataLastAttemptedAt: strings.TrimSpace(in.MetadataLastAttemptedAt),
MetadataError: strings.TrimSpace(in.MetadataError),
}
if len(out.Topics) == 0 {
@@ -59,10 +68,48 @@ func Normalize(in thoughttypes.ThoughtMetadata, capture config.CaptureConfig) th
if out.Source == "" {
out.Source = Fallback(capture).Source
}
if out.MetadataStatus == "" {
out.MetadataStatus = MetadataStatusComplete
}
if out.MetadataStatus == MetadataStatusComplete {
out.MetadataError = ""
}
return out
}
func MarkMetadataPending(base thoughttypes.ThoughtMetadata, capture config.CaptureConfig, attempt time.Time, err error) thoughttypes.ThoughtMetadata {
out := Normalize(base, capture)
out.MetadataStatus = MetadataStatusPending
out.MetadataLastAttemptedAt = attempt.UTC().Format(time.RFC3339)
if err != nil {
out.MetadataError = strings.TrimSpace(err.Error())
}
out.MetadataUpdatedAt = strings.TrimSpace(base.MetadataUpdatedAt)
return out
}
func MarkMetadataFailed(base thoughttypes.ThoughtMetadata, capture config.CaptureConfig, attempt time.Time, err error) thoughttypes.ThoughtMetadata {
out := Normalize(base, capture)
out.MetadataStatus = MetadataStatusFailed
out.MetadataLastAttemptedAt = attempt.UTC().Format(time.RFC3339)
if err != nil {
out.MetadataError = strings.TrimSpace(err.Error())
}
out.MetadataUpdatedAt = strings.TrimSpace(base.MetadataUpdatedAt)
return out
}
func MarkMetadataComplete(base thoughttypes.ThoughtMetadata, capture config.CaptureConfig, updatedAt time.Time) thoughttypes.ThoughtMetadata {
out := Normalize(base, capture)
out.MetadataStatus = MetadataStatusComplete
timestamp := updatedAt.UTC().Format(time.RFC3339)
out.MetadataUpdatedAt = timestamp
out.MetadataLastAttemptedAt = timestamp
out.MetadataError = ""
return out
}
func normalizeList(values []string, limit int) []string {
seen := make(map[string]struct{}, len(values))
result := make([]string, 0, len(values))
@@ -100,6 +147,19 @@ func normalizeType(value string) string {
return DefaultType
}
func normalizeMetadataStatus(value string) string {
switch strings.ToLower(strings.TrimSpace(value)) {
case "", MetadataStatusComplete:
return MetadataStatusComplete
case MetadataStatusPending:
return MetadataStatusPending
case MetadataStatusFailed:
return MetadataStatusFailed
default:
return MetadataStatusComplete
}
}
func normalizeSource(value string) string {
normalized := strings.TrimSpace(value)
if normalized == "" {

57
internal/metadata/normalize_test.go
View File

@@ -3,6 +3,7 @@ package metadata
import (
"strings"
"testing"
"time"
"github.com/google/uuid"
@@ -31,6 +32,9 @@ func TestFallbackUsesConfiguredDefaults(t *testing.T) {
if got.Source != "mcp" {
t.Fatalf("Fallback source = %q, want mcp", got.Source)
}
if got.MetadataStatus != MetadataStatusComplete {
t.Fatalf("Fallback metadata status = %q, want complete", got.MetadataStatus)
}
}
func TestNormalizeTrimsDedupesAndCapsTopics(t *testing.T) {
@@ -102,3 +106,56 @@ func TestNormalizeDedupesAttachmentsByFileID(t *testing.T) {
t.Fatalf("Attachment kind = %q, want image", got.Attachments[0].Kind)
}
}
func TestMarkMetadataPendingTracksAttemptWithoutClearingPreviousSuccess(t *testing.T) {
attempt := time.Date(2026, 3, 30, 10, 0, 0, 0, time.UTC)
base := thoughttypes.ThoughtMetadata{
Topics: []string{"go"},
MetadataUpdatedAt: "2026-03-29T10:00:00Z",
}
got := MarkMetadataPending(base, testCaptureConfig(), attempt, errTestMetadataFailure)
if got.MetadataStatus != MetadataStatusPending {
t.Fatalf("MetadataStatus = %q, want pending", got.MetadataStatus)
}
if got.MetadataUpdatedAt != "2026-03-29T10:00:00Z" {
t.Fatalf("MetadataUpdatedAt = %q, want previous success timestamp", got.MetadataUpdatedAt)
}
if got.MetadataLastAttemptedAt != "2026-03-30T10:00:00Z" {
t.Fatalf("MetadataLastAttemptedAt = %q, want attempt timestamp", got.MetadataLastAttemptedAt)
}
if got.MetadataError == "" {
t.Fatal("MetadataError is empty, want failure message")
}
}
func TestMarkMetadataCompleteClearsErrorAndSetsTimestamps(t *testing.T) {
attempt := time.Date(2026, 3, 30, 10, 0, 0, 0, time.UTC)
got := MarkMetadataComplete(thoughttypes.ThoughtMetadata{
Topics: []string{"go"},
MetadataStatus: MetadataStatusFailed,
MetadataError: "timeout",
}, testCaptureConfig(), attempt)
if got.MetadataStatus != MetadataStatusComplete {
t.Fatalf("MetadataStatus = %q, want complete", got.MetadataStatus)
}
if got.MetadataUpdatedAt != "2026-03-30T10:00:00Z" {
t.Fatalf("MetadataUpdatedAt = %q, want completion timestamp", got.MetadataUpdatedAt)
}
if got.MetadataLastAttemptedAt != "2026-03-30T10:00:00Z" {
t.Fatalf("MetadataLastAttemptedAt = %q, want completion timestamp", got.MetadataLastAttemptedAt)
}
if got.MetadataError != "" {
t.Fatalf("MetadataError = %q, want empty", got.MetadataError)
}
}
var errTestMetadataFailure = testError("timeout")
type testError string
func (e testError) Error() string {
return string(e)
}