feat: implement file upload handler and related functionality

- Added file upload handler to process both multipart and raw file uploads.
- Implemented parsing logic for upload requests, including handling file metadata.
- Introduced SaveFileDecodedInput structure for handling decoded file uploads.
- Created unit tests for file upload parsing and validation.

feat: add metadata retry configuration and functionality

- Introduced MetadataRetryConfig to the application configuration.
- Implemented MetadataRetryer to handle retrying metadata extraction for thoughts.
- Added new tool for retrying failed metadata extractions.
- Updated thought metadata structure to include status and timestamps for metadata processing.

fix: enhance metadata normalization and error handling

- Updated metadata normalization functions to track status and errors.
- Improved handling of metadata extraction failures during thought updates and captures.
- Ensured that metadata status is correctly set during various operations.

refactor: streamline file saving logic in FilesTool

- Refactored Save method in FilesTool to utilize new SaveDecoded method.
- Simplified project and thought ID resolution logic during file saving.

internal/config/config.go

@@ -18,6 +18,7 @@ type Config struct {
 	Logging       LoggingConfig       `yaml:"logging"`
 	Observability ObservabilityConfig `yaml:"observability"`
 	Backfill      BackfillConfig      `yaml:"backfill"`
+	MetadataRetry MetadataRetryConfig `yaml:"metadata_retry"`
 }
 
 type ServerConfig struct {
@@ -152,6 +153,14 @@ type BackfillConfig struct {
 	IncludeArchived bool          `yaml:"include_archived"`
 }
 
+type MetadataRetryConfig struct {
+	Enabled         bool          `yaml:"enabled"`
+	RunOnStartup    bool          `yaml:"run_on_startup"`
+	Interval        time.Duration `yaml:"interval"`
+	MaxPerRun       int           `yaml:"max_per_run"`
+	IncludeArchived bool          `yaml:"include_archived"`
+}
+
 func (c AIMetadataConfig) EffectiveFallbackModels() []string {
 	models := make([]string, 0, len(c.FallbackModels)+1)
 	for _, model := range c.FallbackModels {

internal/config/loader.go

@@ -103,6 +103,12 @@ func defaultConfig() Config {
 			BatchSize:    20,
 			MaxPerRun:    100,
 		},
+		MetadataRetry: MetadataRetryConfig{
+			Enabled:      false,
+			RunOnStartup: false,
+			Interval:     24 * time.Hour,
+			MaxPerRun:    100,
+		},
 	}
 }
 

internal/config/validate.go

@@ -89,6 +89,11 @@ func (c Config) Validate() error {
 			return fmt.Errorf("invalid config: backfill.max_per_run must be >= backfill.batch_size")
 		}
 	}
+	if c.MetadataRetry.Enabled {
+		if c.MetadataRetry.MaxPerRun <= 0 {
+			return fmt.Errorf("invalid config: metadata_retry.max_per_run must be greater than zero when metadata_retry is enabled")
+		}
+	}
 
 	return nil
 }

internal/config/validate_test.go

@@ -111,3 +111,13 @@ func TestValidateRejectsEmptyAuth(t *testing.T) {
 		t.Fatal("Validate() error = nil, want error when neither auth.keys nor auth.oauth.clients is configured")
 	}
 }
+
+func TestValidateRejectsInvalidMetadataRetryConfig(t *testing.T) {
+	cfg := validConfig()
+	cfg.MetadataRetry.Enabled = true
+	cfg.MetadataRetry.MaxPerRun = 0
+
+	if err := cfg.Validate(); err == nil {
+		t.Fatal("Validate() error = nil, want error for invalid metadata retry config")
+	}
+}