test(config): add migration tests for litellm provider

* Implement tests for migrating configuration from v1 to v2 for the litellm provider.
* Validate the structure and values of the migrated configuration.
* Ensure migration rejects newer versions of the configuration.
fix(validate): enhance AI provider validation logic
* Consolidate provider validation into a dedicated method.
* Ensure at least one provider is specified and validate its type.
* Check for required fields based on provider type.
fix(mcpserver): update tool set to use new enrichment tool
* Replace RetryMetadataTool with RetryEnrichmentTool in the ToolSet.
fix(tools): refactor tools to use embedding and metadata runners
* Update tools to utilize EmbeddingRunner and MetadataRunner instead of Provider.
* Adjust method calls to align with the new runner interfaces.

internal/ai/compat/client_test.go

@@ -11,6 +11,17 @@ import (
 	"testing"
 )
 
+func newTestClient(t *testing.T, url string) *Client {
+	t.Helper()
+	return New(Config{
+		Name:       "litellm",
+		BaseURL:    url,
+		APIKey:     "test-key",
+		HTTPClient: http.DefaultClient,
+		Log:        slog.New(slog.NewTextHandler(io.Discard, nil)),
+	})
+}
+
 func TestExtractMetadataFromStreamingResponse(t *testing.T) {
 	t.Parallel()
 
@@ -26,6 +37,9 @@ func TestExtractMetadataFromStreamingResponse(t *testing.T) {
 		if req.Stream == nil || !*req.Stream {
 			t.Fatalf("stream flag = %v, want true", req.Stream)
 		}
+		if req.Model != "qwen3.5:latest" {
+			t.Fatalf("model = %q, want qwen3.5:latest", req.Model)
+		}
 
 		w.Header().Set("Content-Type", "text/event-stream")
 		_, _ = io.WriteString(w, "data: {\"choices\":[{\"delta\":{\"content\":\"{\\\"people\\\":[],\"}}]}\n\n")
@@ -35,20 +49,13 @@ func TestExtractMetadataFromStreamingResponse(t *testing.T) {
 	}))
 	defer server.Close()
 
-	client := New(Config{
-		Name:           "litellm",
-		BaseURL:        server.URL,
-		APIKey:         "test-key",
-		MetadataModel:  "qwen3.5:latest",
-		Temperature:    0.1,
-		HTTPClient:     server.Client(),
-		Log:            slog.New(slog.NewTextHandler(io.Discard, nil)),
-		EmbeddingModel: "unused",
-	})
-
-	metadata, err := client.ExtractMetadata(context.Background(), "Project idea: Build an Android companion app.")
+	client := newTestClient(t, server.URL)
+	metadata, err := client.ExtractMetadataWith(context.Background(), MetadataOptions{
+		Model:       "qwen3.5:latest",
+		Temperature: 0.1,
+	}, "Project idea: Build an Android companion app.")
 	if err != nil {
-		t.Fatalf("ExtractMetadata() error = %v", err)
+		t.Fatalf("ExtractMetadataWith() error = %v", err)
 	}
 
 	if metadata.Type != "idea" {
@@ -94,20 +101,13 @@ func TestExtractMetadataRetriesWithoutJSONMode(t *testing.T) {
 	}))
 	defer server.Close()
 
-	client := New(Config{
-		Name:           "litellm",
-		BaseURL:        server.URL,
-		APIKey:         "test-key",
-		MetadataModel:  "qwen3.5:latest",
-		Temperature:    0.1,
-		HTTPClient:     server.Client(),
-		Log:            slog.New(slog.NewTextHandler(io.Discard, nil)),
-		EmbeddingModel: "unused",
-	})
-
-	metadata, err := client.ExtractMetadata(context.Background(), "Project idea: Build an Android companion app.")
+	client := newTestClient(t, server.URL)
+	metadata, err := client.ExtractMetadataWith(context.Background(), MetadataOptions{
+		Model:       "qwen3.5:latest",
+		Temperature: 0.1,
+	}, "Project idea: Build an Android companion app.")
 	if err != nil {
-		t.Fatalf("ExtractMetadata() error = %v", err)
+		t.Fatalf("ExtractMetadataWith() error = %v", err)
 	}
 
 	if metadata.Type != "idea" {
@@ -127,71 +127,33 @@ func TestExtractMetadataRetriesWithoutJSONMode(t *testing.T) {
 	}
 }
 
-func TestExtractMetadataBypassesInvalidFallbackModelAfterFirstFailure(t *testing.T) {
+func TestIsPermanentModelError(t *testing.T) {
 	t.Parallel()
 
-	var mu sync.Mutex
-	primaryCalls := 0
-	invalidFallbackCalls := 0
-
-	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		defer func() {
-			_ = r.Body.Close()
-		}()
-
-		var req chatCompletionsRequest
-		if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
-			t.Fatalf("decode request: %v", err)
-		}
-
-		switch req.Model {
-		case "empty-primary":
-			_, _ = io.WriteString(w, `{"choices":[{"message":{"role":"assistant","content":""}}]}`)
-		case "qwen3.5:latest":
-			mu.Lock()
-			primaryCalls++
-			mu.Unlock()
-			_, _ = io.WriteString(w, `{"choices":[{"message":{"role":"assistant","content":"{\"people\":[],\"action_items\":[],\"dates_mentioned\":[],\"topics\":[\"metadata\"],\"type\":\"observation\",\"source\":\"primary\"}"}}]}`)
-		case "qwen3":
-			mu.Lock()
-			invalidFallbackCalls++
-			mu.Unlock()
-			w.WriteHeader(http.StatusBadRequest)
-			_, _ = io.WriteString(w, "{\"error\":{\"message\":\"{'error': '/chat/completions: Invalid model name passed in model=qwen3. Call `/v1/models` to view available models for your key.'}\"}}")
-		default:
-			t.Fatalf("unexpected model %q", req.Model)
-		}
-	}))
-	defer server.Close()
-
-	client := New(Config{
-		Name:                   "litellm",
-		BaseURL:                server.URL,
-		APIKey:                 "test-key",
-		MetadataModel:          "empty-primary",
-		FallbackMetadataModels: []string{"qwen3", "qwen3.5:latest"},
-		Temperature:            0.1,
-		HTTPClient:             server.Client(),
-		Log:                    slog.New(slog.NewTextHandler(io.Discard, nil)),
-		EmbeddingModel:         "unused",
-	})
-
-	for i := 0; i < 2; i++ {
-		metadata, err := client.ExtractMetadata(context.Background(), "A short note about metadata.")
-		if err != nil {
-			t.Fatalf("ExtractMetadata() error = %v", err)
-		}
-		if metadata.Source != "primary" {
-			t.Fatalf("metadata source = %q, want primary", metadata.Source)
-		}
+	cases := []struct {
+		name string
+		err  error
+		want bool
+	}{
+		{"nil", nil, false},
+		{"invalid model", errMsg("Invalid model name passed in model=qwen3"), true},
+		{"model not found", errMsg("model_not_found"), true},
+		{"no such model", errMsg("no such model"), true},
+		{"transient", errMsg("connection refused"), false},
 	}
 
-	mu.Lock()
-	defer mu.Unlock()
-	if invalidFallbackCalls != 1 {
-		t.Fatalf("invalid fallback calls = %d, want 1", invalidFallbackCalls)
-	}
-	if primaryCalls != 2 {
-		t.Fatalf("valid fallback calls = %d, want 2", primaryCalls)
+	for _, tc := range cases {
+		tc := tc
+		t.Run(tc.name, func(t *testing.T) {
+			if got := IsPermanentModelError(tc.err); got != tc.want {
+				t.Fatalf("IsPermanentModelError(%v) = %v, want %v", tc.err, got, tc.want)
+			}
+		})
 	}
 }
+
+type stringError string
+
+func (s stringError) Error() string { return string(s) }
+
+func errMsg(s string) error { return stringError(s) }