feat(config): add fallback model support for AI configurations

configs/config.example.yaml

@@ -42,6 +42,7 @@ ai:
     dimensions: 1536
   metadata:
     model: "gpt-4o-mini"
+    fallback_model: ""
     temperature: 0.1
   litellm:
     base_url: "http://localhost:4000/v1"
@@ -50,6 +51,7 @@ ai:
     request_headers: {}
     embedding_model: "openrouter/openai/text-embedding-3-small"
     metadata_model: "gpt-4o-mini"
+    fallback_metadata_model: ""
   ollama:
     base_url: "http://localhost:11434/v1"
     api_key: "ollama"

internal/ai/compat/client.go

@@ -38,6 +38,7 @@ type Client struct {
 	apiKey                string
 	embeddingModel        string
 	metadataModel         string
+	fallbackMetadataModel string
 	temperature           float64
 	headers               map[string]string
 	httpClient            *http.Client
@@ -51,6 +52,7 @@ type Config struct {
 	APIKey                string
 	EmbeddingModel        string
 	MetadataModel         string
+	FallbackMetadataModel string
 	Temperature           float64
 	Headers               map[string]string
 	HTTPClient            *http.Client
@@ -105,6 +107,7 @@ func New(cfg Config) *Client {
 		apiKey:                cfg.APIKey,
 		embeddingModel:        cfg.EmbeddingModel,
 		metadataModel:         cfg.MetadataModel,
+		fallbackMetadataModel: cfg.FallbackMetadataModel,
 		temperature:           cfg.Temperature,
 		headers:               cfg.Headers,
 		httpClient:            cfg.HTTPClient,
@@ -146,8 +149,24 @@ func (c *Client) ExtractMetadata(ctx context.Context, input string) (thoughttype
 		return thoughttypes.ThoughtMetadata{}, fmt.Errorf("%s extract metadata: input must not be empty", c.name)
 	}
 
+	result, err := c.extractMetadataWithModel(ctx, input, c.metadataModel)
+	if err != nil && c.fallbackMetadataModel != "" && ctx.Err() == nil {
+		if c.log != nil {
+			c.log.Warn("metadata extraction failed, trying fallback model",
+				slog.String("provider", c.name),
+				slog.String("primary_model", c.metadataModel),
+				slog.String("fallback_model", c.fallbackMetadataModel),
+				slog.String("error", err.Error()),
+			)
+		}
+		return c.extractMetadataWithModel(ctx, input, c.fallbackMetadataModel)
+	}
+	return result, err
+}
+
+func (c *Client) extractMetadataWithModel(ctx context.Context, input, model string) (thoughttypes.ThoughtMetadata, error) {
 	req := chatCompletionsRequest{
-		Model:       c.metadataModel,
+		Model:       model,
 		Temperature: c.temperature,
 		ResponseFormat: &responseType{
 			Type: "json_object",
@@ -172,6 +191,9 @@ func (c *Client) ExtractMetadata(ctx context.Context, input string) (thoughttype
 	metadataText := strings.TrimSpace(resp.Choices[0].Message.Content)
 	metadataText = stripCodeFence(metadataText)
 	metadataText = extractJSONObject(metadataText)
+	if metadataText == "" {
+		return thoughttypes.ThoughtMetadata{}, fmt.Errorf("%s metadata: response contains no JSON object", c.name)
+	}
 
 	var metadata thoughttypes.ThoughtMetadata
 	if err := json.Unmarshal([]byte(metadataText), &metadata); err != nil {
@@ -241,7 +263,7 @@ func (c *Client) doJSON(ctx context.Context, path string, requestBody any, dest
 		resp, err := c.httpClient.Do(req)
 		if err != nil {
 			lastErr = fmt.Errorf("%s request failed: %w", c.name, err)
-			if attempt < maxAttempts && isRetryableError(err) {
+			if attempt < maxAttempts && ctx.Err() == nil && isRetryableError(err) {
 				if retryErr := sleepRetry(ctx, attempt, c.log, c.name); retryErr != nil {
 					return retryErr
 				}
@@ -293,7 +315,7 @@ func extractJSONObject(s string) string {
 	start := strings.Index(s, "{")
 	end := strings.LastIndex(s, "}")
 	if start == -1 || end == -1 || end <= start {
-		return s
+		return ""
 	}
 	return s[start : end+1]
 }

internal/ai/litellm/client.go

@@ -9,12 +9,17 @@ import (
 )
 
 func New(cfg config.AIConfig, httpClient *http.Client, log *slog.Logger) (*compat.Client, error) {
+	fallback := cfg.LiteLLM.FallbackMetadataModel
+	if fallback == "" {
+		fallback = cfg.Metadata.FallbackModel
+	}
 	return compat.New(compat.Config{
 		Name:                  "litellm",
 		BaseURL:               cfg.LiteLLM.BaseURL,
 		APIKey:                cfg.LiteLLM.APIKey,
 		EmbeddingModel:        cfg.LiteLLM.EmbeddingModel,
 		MetadataModel:         cfg.LiteLLM.MetadataModel,
+		FallbackMetadataModel: fallback,
 		Temperature:           cfg.Metadata.Temperature,
 		Headers:               cfg.LiteLLM.RequestHeaders,
 		HTTPClient:            httpClient,

internal/ai/ollama/client.go

@@ -15,6 +15,7 @@ func New(cfg config.AIConfig, httpClient *http.Client, log *slog.Logger) (*compa
 		APIKey:                cfg.Ollama.APIKey,
 		EmbeddingModel:        cfg.Embeddings.Model,
 		MetadataModel:         cfg.Metadata.Model,
+		FallbackMetadataModel: cfg.Metadata.FallbackModel,
 		Temperature:           cfg.Metadata.Temperature,
 		Headers:               cfg.Ollama.RequestHeaders,
 		HTTPClient:            httpClient,

internal/ai/openrouter/client.go

@@ -26,6 +26,7 @@ func New(cfg config.AIConfig, httpClient *http.Client, log *slog.Logger) (*compa
 		APIKey:                cfg.OpenRouter.APIKey,
 		EmbeddingModel:        cfg.Embeddings.Model,
 		MetadataModel:         cfg.Metadata.Model,
+		FallbackMetadataModel: cfg.Metadata.FallbackModel,
 		Temperature:           cfg.Metadata.Temperature,
 		Headers:               headers,
 		HTTPClient:            httpClient,

internal/config/config.go

@@ -85,6 +85,7 @@ type AIEmbeddingConfig struct {
 
 type AIMetadataConfig struct {
 	Model         string  `yaml:"model"`
+	FallbackModel string  `yaml:"fallback_model"`
 	Temperature   float64 `yaml:"temperature"`
 }
 
@@ -95,6 +96,7 @@ type LiteLLMConfig struct {
 	RequestHeaders        map[string]string `yaml:"request_headers"`
 	EmbeddingModel        string            `yaml:"embedding_model"`
 	MetadataModel         string            `yaml:"metadata_model"`
+	FallbackMetadataModel string            `yaml:"fallback_metadata_model"`
 }
 
 type OllamaConfig struct {