mirror of
https://github.com/Warky-Devs/vecna.git
synced 2026-08-07 10:47:38 +00:00
feat(server): add passthrough proxy for OpenAI-compatible endpoints
* implement proxy handler for various OpenAI API routes * add error handling for request body and response streaming * introduce new error response format for API compatibility * add tests for recover middleware to handle panics gracefully
This commit is contained in:
@@ -23,7 +23,7 @@ func (h *handler) googleDispatch(w http.ResponseWriter, req bunrouter.Request) e
|
||||
func (h *handler) googleDispatchMapped(w http.ResponseWriter, req bunrouter.Request) error {
|
||||
em, err := h.resolveExtraMap(req.Param("mapping"))
|
||||
if err != nil {
|
||||
return writeJSON(w, http.StatusNotFound, map[string]string{"error": err.Error()})
|
||||
return writeError(w, http.StatusNotFound, err.Error())
|
||||
}
|
||||
return h.googleDispatchWithAdapter(w, req, em.Adapter, em.ForwardTarget)
|
||||
}
|
||||
@@ -32,7 +32,7 @@ func (h *handler) googleDispatchWithAdapter(w http.ResponseWriter, req bunrouter
|
||||
modelaction := req.Param("modelaction") // e.g. "text-embedding-foo:embedContent"
|
||||
idx := strings.LastIndex(modelaction, ":")
|
||||
if idx < 0 {
|
||||
return writeJSON(w, http.StatusNotFound, map[string]string{"error": "invalid Google API path"})
|
||||
return writeError(w, http.StatusNotFound, "invalid Google API path")
|
||||
}
|
||||
model := modelaction[:idx]
|
||||
action := modelaction[idx+1:]
|
||||
@@ -46,7 +46,7 @@ func (h *handler) googleDispatchWithAdapter(w http.ResponseWriter, req bunrouter
|
||||
case "batchEmbedContents":
|
||||
return h.googleBatchEmbedContentsWithAdapter(w, req, adp, targetOverride)
|
||||
default:
|
||||
return writeJSON(w, http.StatusNotFound, map[string]string{"error": "unknown Google API method: " + action})
|
||||
return writeError(w, http.StatusNotFound, "unknown Google API method: "+action)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -78,7 +78,7 @@ func (h *handler) googleEmbedContentWithAdapter(w http.ResponseWriter, req bunro
|
||||
|
||||
var body googleEmbedContentRequest
|
||||
if err := json.NewDecoder(req.Body).Decode(&body); err != nil {
|
||||
return writeJSON(w, http.StatusBadRequest, map[string]string{"error": "invalid request body"})
|
||||
return writeError(w, http.StatusBadRequest, "invalid request body")
|
||||
}
|
||||
|
||||
texts := make([]string, len(body.Content.Parts))
|
||||
@@ -95,7 +95,7 @@ func (h *handler) googleEmbedContentWithAdapter(w http.ResponseWriter, req bunro
|
||||
embedResp, err := client.Embed(req.Context(), embedclient.Request{Texts: texts, Model: model})
|
||||
trace.ForwardDuration = time.Since(t0)
|
||||
if err != nil {
|
||||
return writeJSON(w, http.StatusBadGateway, map[string]string{"error": err.Error()})
|
||||
return writeError(w, http.StatusBadGateway, err.Error())
|
||||
}
|
||||
trace.ForwardModel = embedResp.Model
|
||||
trace.PromptTokens = embedResp.Usage.PromptTokens
|
||||
@@ -106,7 +106,7 @@ func (h *handler) googleEmbedContentWithAdapter(w http.ResponseWriter, req bunro
|
||||
if len(embedResp.Embeddings) > 0 {
|
||||
adapted, err = adp.Adapt(embedResp.Embeddings[0])
|
||||
if err != nil {
|
||||
return writeJSON(w, http.StatusInternalServerError, map[string]string{"error": err.Error()})
|
||||
return writeError(w, http.StatusInternalServerError, err.Error())
|
||||
}
|
||||
}
|
||||
trace.TranslateDuration = time.Since(t1)
|
||||
@@ -133,7 +133,7 @@ func (h *handler) googleBatchEmbedContentsWithAdapter(w http.ResponseWriter, req
|
||||
|
||||
var body googleBatchRequest
|
||||
if err := json.NewDecoder(req.Body).Decode(&body); err != nil {
|
||||
return writeJSON(w, http.StatusBadRequest, map[string]string{"error": "invalid request body"})
|
||||
return writeError(w, http.StatusBadRequest, "invalid request body")
|
||||
}
|
||||
|
||||
var texts []string
|
||||
@@ -152,7 +152,7 @@ func (h *handler) googleBatchEmbedContentsWithAdapter(w http.ResponseWriter, req
|
||||
embedResp, err := client.Embed(req.Context(), embedclient.Request{Texts: texts, Model: model})
|
||||
trace.ForwardDuration = time.Since(t0)
|
||||
if err != nil {
|
||||
return writeJSON(w, http.StatusBadGateway, map[string]string{"error": err.Error()})
|
||||
return writeError(w, http.StatusBadGateway, err.Error())
|
||||
}
|
||||
trace.ForwardModel = embedResp.Model
|
||||
trace.PromptTokens = embedResp.Usage.PromptTokens
|
||||
@@ -163,7 +163,7 @@ func (h *handler) googleBatchEmbedContentsWithAdapter(w http.ResponseWriter, req
|
||||
for i, vec := range embedResp.Embeddings {
|
||||
adapted, adaptErr := adp.Adapt(vec)
|
||||
if adaptErr != nil {
|
||||
return writeJSON(w, http.StatusInternalServerError, map[string]string{"error": adaptErr.Error()})
|
||||
return writeError(w, http.StatusInternalServerError, adaptErr.Error())
|
||||
}
|
||||
result[i] = googleEmbeddingValues{Values: adapted}
|
||||
}
|
||||
|
||||
+73
-5
@@ -12,6 +12,7 @@ import (
|
||||
"github.com/Warky-Devs/vecna.git/pkg/adapter"
|
||||
"github.com/Warky-Devs/vecna.git/pkg/config"
|
||||
"github.com/Warky-Devs/vecna.git/pkg/embedclient"
|
||||
"github.com/Warky-Devs/vecna.git/pkg/forwardproxy"
|
||||
)
|
||||
|
||||
// ExtraMap pairs a dimension adapter with an optional forward-target override.
|
||||
@@ -22,11 +23,12 @@ type ExtraMap struct {
|
||||
|
||||
// handler holds shared dependencies for all HTTP handlers.
|
||||
type handler struct {
|
||||
cfg *config.Config
|
||||
clients map[string]embedclient.Client
|
||||
adapter adapter.Adapter
|
||||
extraMaps map[string]ExtraMap
|
||||
logger *zap.Logger
|
||||
cfg *config.Config
|
||||
clients map[string]embedclient.Client
|
||||
adapter adapter.Adapter
|
||||
extraMaps map[string]ExtraMap
|
||||
forwarders map[string]*forwardproxy.Router
|
||||
logger *zap.Logger
|
||||
}
|
||||
|
||||
// resolveExtraMap returns the ExtraMap for the named extra_map entry.
|
||||
@@ -66,6 +68,33 @@ func (h *handler) resolveClientOverride(targetOverride, model string) (embedclie
|
||||
return c, targetOverride, firstEndpointURL(h.cfg, targetOverride)
|
||||
}
|
||||
|
||||
// resolveForwarder selects the raw forwarder for the given model, falling back
|
||||
// to forward.default when the model doesn't name a configured target directly.
|
||||
func (h *handler) resolveForwarder(model string) (*forwardproxy.Router, string, error) {
|
||||
if f, ok := h.forwarders[model]; ok {
|
||||
return f, model, nil
|
||||
}
|
||||
name := h.cfg.Forward.Default
|
||||
f, ok := h.forwarders[name]
|
||||
if !ok {
|
||||
return nil, name, fmt.Errorf("no forward target configured for model %q and no default", model)
|
||||
}
|
||||
return f, name, nil
|
||||
}
|
||||
|
||||
// resolveForwarderOverride selects the forwarder for targetOverride when set,
|
||||
// otherwise falls back to model-based resolution.
|
||||
func (h *handler) resolveForwarderOverride(targetOverride, model string) (*forwardproxy.Router, string, error) {
|
||||
if targetOverride == "" {
|
||||
return h.resolveForwarder(model)
|
||||
}
|
||||
f, ok := h.forwarders[targetOverride]
|
||||
if !ok {
|
||||
return nil, targetOverride, fmt.Errorf("extra_map forward_target %q not configured", targetOverride)
|
||||
}
|
||||
return f, targetOverride, nil
|
||||
}
|
||||
|
||||
func firstEndpointURL(cfg *config.Config, targetName string) string {
|
||||
t, ok := cfg.Forward.Targets[targetName]
|
||||
if !ok || len(t.Endpoints) == 0 {
|
||||
@@ -84,6 +113,45 @@ func writeJSON(w http.ResponseWriter, status int, v interface{}) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// apiError is the OpenAI-compatible error envelope, so existing OpenAI SDKs
|
||||
// (which read err.error.message) surface vecna's own errors correctly instead
|
||||
// of failing to parse a flat {"error": "..."} string.
|
||||
type apiError struct {
|
||||
Message string `json:"message"`
|
||||
Type string `json:"type"`
|
||||
Code int `json:"code"`
|
||||
}
|
||||
|
||||
type apiErrorEnvelope struct {
|
||||
Error apiError `json:"error"`
|
||||
}
|
||||
|
||||
// writeError writes an OpenAI-compatible error response.
|
||||
func writeError(w http.ResponseWriter, status int, message string) error {
|
||||
return writeJSON(w, status, apiErrorEnvelope{Error: apiError{
|
||||
Message: message,
|
||||
Type: errorType(status),
|
||||
Code: status,
|
||||
}})
|
||||
}
|
||||
|
||||
// errorType maps an HTTP status to an OpenAI-style error type string.
|
||||
func errorType(status int) string {
|
||||
switch status {
|
||||
case http.StatusUnauthorized:
|
||||
return "authentication_error"
|
||||
case http.StatusBadRequest, http.StatusNotFound:
|
||||
return "invalid_request_error"
|
||||
case http.StatusBadGateway:
|
||||
return "upstream_error"
|
||||
default:
|
||||
if status >= 500 {
|
||||
return "internal_error"
|
||||
}
|
||||
return "api_error"
|
||||
}
|
||||
}
|
||||
|
||||
// writeTraceHeaders writes X-Vecna-* timing headers from the RequestTrace.
|
||||
func writeTraceHeaders(w http.ResponseWriter, t *RequestTrace) {
|
||||
total := time.Since(t.Start)
|
||||
|
||||
+34
-7
@@ -42,32 +42,59 @@ func (h *handler) openAIEmbeddings(w http.ResponseWriter, req bunrouter.Request)
|
||||
func (h *handler) openAIEmbeddingsMapped(w http.ResponseWriter, req bunrouter.Request) error {
|
||||
em, err := h.resolveExtraMap(req.Param("mapping"))
|
||||
if err != nil {
|
||||
return writeJSON(w, http.StatusNotFound, map[string]string{"error": err.Error()})
|
||||
return writeError(w, http.StatusNotFound, err.Error())
|
||||
}
|
||||
return h.openAIEmbeddingsWithAdapter(w, req, em.Adapter, em.ForwardTarget)
|
||||
}
|
||||
|
||||
// openAIEmbeddingsGet is a GET convenience variant (e.g. for browser/curl testing):
|
||||
// ?input=foo&input=bar&model=name in place of a JSON body.
|
||||
func (h *handler) openAIEmbeddingsGet(w http.ResponseWriter, req bunrouter.Request) error {
|
||||
return h.openAIEmbeddingsGetWithAdapter(w, req, h.adapter, "")
|
||||
}
|
||||
|
||||
func (h *handler) openAIEmbeddingsGetMapped(w http.ResponseWriter, req bunrouter.Request) error {
|
||||
em, err := h.resolveExtraMap(req.Param("mapping"))
|
||||
if err != nil {
|
||||
return writeError(w, http.StatusNotFound, err.Error())
|
||||
}
|
||||
return h.openAIEmbeddingsGetWithAdapter(w, req, em.Adapter, em.ForwardTarget)
|
||||
}
|
||||
|
||||
func (h *handler) openAIEmbeddingsGetWithAdapter(w http.ResponseWriter, req bunrouter.Request, adp adapter.Adapter, targetOverride string) error {
|
||||
q := req.URL.Query()
|
||||
texts := q["input"]
|
||||
if len(texts) == 0 {
|
||||
return writeError(w, http.StatusBadRequest, "query parameter \"input\" is required")
|
||||
}
|
||||
return h.processEmbeddings(w, req, adp, targetOverride, texts, q.Get("model"))
|
||||
}
|
||||
|
||||
func (h *handler) openAIEmbeddingsWithAdapter(w http.ResponseWriter, req bunrouter.Request, adp adapter.Adapter, targetOverride string) error {
|
||||
var body openAIEmbedRequest
|
||||
if err := json.NewDecoder(req.Body).Decode(&body); err != nil {
|
||||
return writeJSON(w, http.StatusBadRequest, map[string]string{"error": "invalid request body"})
|
||||
return writeError(w, http.StatusBadRequest, "invalid request body")
|
||||
}
|
||||
|
||||
texts, err := toStringSlice(body.Input)
|
||||
if err != nil {
|
||||
return writeJSON(w, http.StatusBadRequest, map[string]string{"error": err.Error()})
|
||||
return writeError(w, http.StatusBadRequest, err.Error())
|
||||
}
|
||||
|
||||
client, targetName, targetURL := h.resolveClientOverride(targetOverride, body.Model)
|
||||
return h.processEmbeddings(w, req, adp, targetOverride, texts, body.Model)
|
||||
}
|
||||
|
||||
func (h *handler) processEmbeddings(w http.ResponseWriter, req bunrouter.Request, adp adapter.Adapter, targetOverride string, texts []string, model string) error {
|
||||
client, targetName, targetURL := h.resolveClientOverride(targetOverride, model)
|
||||
trace := TraceFromContext(req.Context())
|
||||
trace.ForwardTarget = targetName
|
||||
trace.ForwardURL = targetURL
|
||||
|
||||
t0 := time.Now()
|
||||
embedResp, err := client.Embed(req.Context(), embedclient.Request{Texts: texts, Model: body.Model})
|
||||
embedResp, err := client.Embed(req.Context(), embedclient.Request{Texts: texts, Model: model})
|
||||
trace.ForwardDuration = time.Since(t0)
|
||||
if err != nil {
|
||||
return writeJSON(w, http.StatusBadGateway, map[string]string{"error": err.Error()})
|
||||
return writeError(w, http.StatusBadGateway, err.Error())
|
||||
}
|
||||
trace.ForwardModel = embedResp.Model
|
||||
trace.PromptTokens = embedResp.Usage.PromptTokens
|
||||
@@ -78,7 +105,7 @@ func (h *handler) openAIEmbeddingsWithAdapter(w http.ResponseWriter, req bunrout
|
||||
for i, vec := range embedResp.Embeddings {
|
||||
adapted, adaptErr := adp.Adapt(vec)
|
||||
if adaptErr != nil {
|
||||
return writeJSON(w, http.StatusInternalServerError, map[string]string{"error": adaptErr.Error()})
|
||||
return writeError(w, http.StatusInternalServerError, adaptErr.Error())
|
||||
}
|
||||
data[i] = openAIEmbedDatum{Object: "embedding", Embedding: adapted, Index: i}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,182 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"io"
|
||||
"net/http"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/uptrace/bunrouter"
|
||||
|
||||
"github.com/Warky-Devs/vecna.git/pkg/forwardproxy"
|
||||
)
|
||||
|
||||
// passthroughRoute describes one OpenAI-compatible endpoint forwarded verbatim.
|
||||
type passthroughRoute struct {
|
||||
method string
|
||||
path string // route pattern registered with bunrouter (":model" etc.)
|
||||
}
|
||||
|
||||
// passthroughRoutes lists the OpenAI-compatible endpoints vecna forwards without
|
||||
// any body processing, beyond /v1/embeddings (which gets dimension adaptation).
|
||||
var passthroughRoutes = []passthroughRoute{
|
||||
{http.MethodGet, "/v1/models"},
|
||||
{http.MethodGet, "/v1/models/:model"},
|
||||
{http.MethodPost, "/v1/chat/completions"},
|
||||
{http.MethodPost, "/v1/completions"},
|
||||
{http.MethodPost, "/v1/moderations"},
|
||||
{http.MethodPost, "/v1/images/generations"},
|
||||
{http.MethodPost, "/v1/images/edits"},
|
||||
{http.MethodPost, "/v1/images/variations"},
|
||||
{http.MethodPost, "/v1/audio/speech"},
|
||||
{http.MethodPost, "/v1/audio/transcriptions"},
|
||||
{http.MethodPost, "/v1/audio/translations"},
|
||||
{http.MethodPost, "/v1/rerank"},
|
||||
}
|
||||
|
||||
// registerPassthroughRoutes wires up both the unmapped and /map/:mapping variants
|
||||
// of every entry in passthroughRoutes.
|
||||
func registerPassthroughRoutes(authed *bunrouter.Group, h *handler) {
|
||||
for _, r := range passthroughRoutes {
|
||||
upstreamPath := strings.Replace(r.path, ":model", "{model}", 1)
|
||||
authed.Handle(r.method, r.path, h.proxyHandler(r.method, upstreamPath))
|
||||
authed.Handle(r.method, "/map/:mapping"+r.path, h.proxyHandlerMapped(r.method, upstreamPath))
|
||||
}
|
||||
}
|
||||
|
||||
// proxyHandler returns a handler that forwards method/path verbatim to the
|
||||
// target resolved from the request body's "model" field (or forward.default).
|
||||
// No vecna-specific processing (adaptation, etc.) is applied to the body.
|
||||
func (h *handler) proxyHandler(method, path string) bunrouter.HandlerFunc {
|
||||
return func(w http.ResponseWriter, req bunrouter.Request) error {
|
||||
return h.doProxy(w, req, method, path, "")
|
||||
}
|
||||
}
|
||||
|
||||
// proxyHandlerMapped is the /map/:mapping variant: the target is always the
|
||||
// extra_map's forward_target, regardless of the body's "model" field.
|
||||
func (h *handler) proxyHandlerMapped(method, path string) bunrouter.HandlerFunc {
|
||||
return func(w http.ResponseWriter, req bunrouter.Request) error {
|
||||
em, err := h.resolveExtraMap(req.Param("mapping"))
|
||||
if err != nil {
|
||||
return writeError(w, http.StatusNotFound, err.Error())
|
||||
}
|
||||
return h.doProxy(w, req, method, path, em.ForwardTarget)
|
||||
}
|
||||
}
|
||||
|
||||
func (h *handler) doProxy(w http.ResponseWriter, req bunrouter.Request, method, pathTemplate, targetOverride string) error {
|
||||
model := req.Param("model")
|
||||
upstreamPath := strings.Replace(pathTemplate, "{model}", model, 1)
|
||||
|
||||
var body []byte
|
||||
if method != http.MethodGet && method != http.MethodDelete {
|
||||
b, err := io.ReadAll(req.Body)
|
||||
if err != nil {
|
||||
return writeError(w, http.StatusBadRequest, "invalid request body")
|
||||
}
|
||||
body = b
|
||||
if model == "" && strings.HasPrefix(req.Header.Get("Content-Type"), "application/json") {
|
||||
model = modelPeek(body)
|
||||
}
|
||||
}
|
||||
|
||||
fwd, targetName, err := h.resolveForwarderOverride(targetOverride, model)
|
||||
if err != nil {
|
||||
return writeError(w, http.StatusBadGateway, err.Error())
|
||||
}
|
||||
|
||||
trace := TraceFromContext(req.Context())
|
||||
trace.ForwardTarget = targetName
|
||||
|
||||
if isStreaming(body) {
|
||||
return h.streamProxy(w, req, fwd, method, upstreamPath, body, trace)
|
||||
}
|
||||
|
||||
t0 := time.Now()
|
||||
res, err := fwd.Do(req.Context(), method, upstreamPath, body, req.Header)
|
||||
trace.ForwardDuration = time.Since(t0)
|
||||
if err != nil {
|
||||
return writeError(w, http.StatusBadGateway, err.Error())
|
||||
}
|
||||
trace.ForwardURL = res.Target
|
||||
|
||||
writeTraceHeaders(w, trace)
|
||||
writeUpstreamResult(w, res)
|
||||
return nil
|
||||
}
|
||||
|
||||
// streamProxy relays a single-attempt streaming response (e.g. SSE chat
|
||||
// completions) directly to the client as bytes arrive.
|
||||
func (h *handler) streamProxy(w http.ResponseWriter, req bunrouter.Request, fwd *forwardproxy.Router, method, path string, body []byte, trace *RequestTrace) error {
|
||||
t0 := time.Now()
|
||||
status, header, respBody, err := fwd.DoStream(req.Context(), method, path, body, req.Header)
|
||||
trace.ForwardDuration = time.Since(t0)
|
||||
if err != nil {
|
||||
return writeError(w, http.StatusBadGateway, err.Error())
|
||||
}
|
||||
defer func() { _ = respBody.Close() }()
|
||||
|
||||
dst := w.Header()
|
||||
for k, vv := range header {
|
||||
if forwardproxy.SkipResponseHeader(k) {
|
||||
continue
|
||||
}
|
||||
for _, v := range vv {
|
||||
dst.Add(k, v)
|
||||
}
|
||||
}
|
||||
w.WriteHeader(status)
|
||||
|
||||
flusher, _ := w.(http.Flusher)
|
||||
buf := make([]byte, 4096)
|
||||
for {
|
||||
n, rerr := respBody.Read(buf)
|
||||
if n > 0 {
|
||||
if _, werr := w.Write(buf[:n]); werr != nil {
|
||||
return nil
|
||||
}
|
||||
if flusher != nil {
|
||||
flusher.Flush()
|
||||
}
|
||||
}
|
||||
if rerr != nil {
|
||||
break
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func writeUpstreamResult(w http.ResponseWriter, res forwardproxy.Result) {
|
||||
dst := w.Header()
|
||||
for k, vv := range res.Header {
|
||||
if forwardproxy.SkipResponseHeader(k) {
|
||||
continue
|
||||
}
|
||||
for _, v := range vv {
|
||||
dst.Add(k, v)
|
||||
}
|
||||
}
|
||||
w.WriteHeader(res.Status)
|
||||
_, _ = w.Write(res.Body)
|
||||
}
|
||||
|
||||
// modelPeek extracts the "model" field from a JSON body without full decoding,
|
||||
// used to select a forward target the same way embeddings does.
|
||||
func modelPeek(body []byte) string {
|
||||
var v struct {
|
||||
Model string `json:"model"`
|
||||
}
|
||||
_ = json.Unmarshal(body, &v)
|
||||
return v.Model
|
||||
}
|
||||
|
||||
// isStreaming reports whether a JSON request body sets "stream": true.
|
||||
func isStreaming(body []byte) bool {
|
||||
var v struct {
|
||||
Stream bool `json:"stream"`
|
||||
}
|
||||
_ = json.Unmarshal(body, &v)
|
||||
return v.Stream
|
||||
}
|
||||
@@ -0,0 +1,88 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
|
||||
"github.com/uptrace/bunrouter"
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
||||
// TestRecoverMiddlewarePanicBeforeResponse verifies a panic in a handler that
|
||||
// hasn't written anything yet is converted into a 500 JSON error instead of
|
||||
// crashing the request goroutine (which would otherwise surface to the client
|
||||
// as a connection reset).
|
||||
func TestRecoverMiddlewarePanicBeforeResponse(t *testing.T) {
|
||||
mw := recoverMiddleware(zap.NewNop())
|
||||
handler := mw(func(w http.ResponseWriter, req bunrouter.Request) error {
|
||||
panic("boom")
|
||||
})
|
||||
|
||||
req := httptest.NewRequestWithContext(context.Background(), http.MethodGet, "/v1/models", nil)
|
||||
w := httptest.NewRecorder()
|
||||
|
||||
err := handler(w, bunrouter.Request{Request: req})
|
||||
if err != nil {
|
||||
t.Fatalf("expected recoverMiddleware to swallow the panic and return nil, got %v", err)
|
||||
}
|
||||
if w.Code != http.StatusInternalServerError {
|
||||
t.Fatalf("expected status 500, got %d", w.Code)
|
||||
}
|
||||
|
||||
var body apiErrorEnvelope
|
||||
if err := json.NewDecoder(w.Body).Decode(&body); err != nil {
|
||||
t.Fatalf("expected a valid JSON error body, got decode error: %v (body=%q)", err, w.Body.String())
|
||||
}
|
||||
if body.Error.Type != "internal_error" {
|
||||
t.Fatalf("expected error type internal_error, got %q", body.Error.Type)
|
||||
}
|
||||
}
|
||||
|
||||
// TestRecoverMiddlewarePanicAfterResponse verifies a panic that happens after
|
||||
// the response has already been committed (e.g. mid-stream) doesn't attempt a
|
||||
// second WriteHeader — it just surfaces as an error for the logger.
|
||||
func TestRecoverMiddlewarePanicAfterResponse(t *testing.T) {
|
||||
mw := recoverMiddleware(zap.NewNop())
|
||||
handler := mw(func(w http.ResponseWriter, req bunrouter.Request) error {
|
||||
w.WriteHeader(http.StatusOK)
|
||||
_, _ = w.Write([]byte("partial"))
|
||||
panic("boom mid-stream")
|
||||
})
|
||||
|
||||
req := httptest.NewRequestWithContext(context.Background(), http.MethodGet, "/v1/chat/completions", nil)
|
||||
w := httptest.NewRecorder()
|
||||
|
||||
err := handler(w, bunrouter.Request{Request: req})
|
||||
if err == nil {
|
||||
t.Fatal("expected an error to be returned once the response was already committed")
|
||||
}
|
||||
if w.Code != http.StatusOK {
|
||||
t.Fatalf("expected the original 200 to stand, got %d", w.Code)
|
||||
}
|
||||
if w.Body.String() != "partial" {
|
||||
t.Fatalf("expected body to be left as-is, got %q", w.Body.String())
|
||||
}
|
||||
}
|
||||
|
||||
// TestRecoverMiddlewareNoPanic verifies the happy path is unaffected.
|
||||
func TestRecoverMiddlewareNoPanic(t *testing.T) {
|
||||
mw := recoverMiddleware(zap.NewNop())
|
||||
handler := mw(func(w http.ResponseWriter, req bunrouter.Request) error {
|
||||
w.WriteHeader(http.StatusOK)
|
||||
_, _ = w.Write([]byte("ok"))
|
||||
return nil
|
||||
})
|
||||
|
||||
req := httptest.NewRequestWithContext(context.Background(), http.MethodGet, "/v1/models", nil)
|
||||
w := httptest.NewRecorder()
|
||||
|
||||
if err := handler(w, bunrouter.Request{Request: req}); err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
if w.Code != http.StatusOK || w.Body.String() != "ok" {
|
||||
t.Fatalf("unexpected response: status=%d body=%q", w.Code, w.Body.String())
|
||||
}
|
||||
}
|
||||
+99
-16
@@ -13,6 +13,7 @@ import (
|
||||
"github.com/Warky-Devs/vecna.git/pkg/adapter"
|
||||
"github.com/Warky-Devs/vecna.git/pkg/config"
|
||||
"github.com/Warky-Devs/vecna.git/pkg/embedclient"
|
||||
"github.com/Warky-Devs/vecna.git/pkg/forwardproxy"
|
||||
"github.com/Warky-Devs/vecna.git/pkg/metrics"
|
||||
"github.com/Warky-Devs/vecna.git/pkg/server/spec"
|
||||
)
|
||||
@@ -24,6 +25,7 @@ func New(
|
||||
clients map[string]embedclient.Client,
|
||||
adp adapter.Adapter,
|
||||
extraMaps map[string]ExtraMap,
|
||||
forwarders map[string]*forwardproxy.Router,
|
||||
reg *metrics.Registry,
|
||||
logger *zap.Logger,
|
||||
) (router *bunrouter.Router, err error) {
|
||||
@@ -35,12 +37,13 @@ func New(
|
||||
}()
|
||||
|
||||
router = bunrouter.New(
|
||||
bunrouter.WithMiddleware(recoverMiddleware(logger)),
|
||||
bunrouter.WithMiddleware(traceMiddleware()),
|
||||
bunrouter.WithMiddleware(metricsMiddleware(reg, adp)),
|
||||
bunrouter.WithMiddleware(loggingMiddleware(logger)),
|
||||
)
|
||||
|
||||
h := &handler{cfg: cfg, clients: clients, adapter: adp, extraMaps: extraMaps, logger: logger}
|
||||
h := &handler{cfg: cfg, clients: clients, adapter: adp, extraMaps: extraMaps, forwarders: forwarders, logger: logger}
|
||||
|
||||
// Public routes — no authentication required.
|
||||
router.GET("/", spec.DocsHandler())
|
||||
@@ -51,6 +54,7 @@ func New(
|
||||
authed := router.NewGroup("", bunrouter.WithMiddleware(authMiddleware(cfg.Server.APIKeys)))
|
||||
|
||||
authed.POST("/v1/embeddings", h.openAIEmbeddings)
|
||||
authed.GET("/v1/embeddings", h.openAIEmbeddingsGet)
|
||||
// Google API uses a literal colon as a method separator (e.g. /v1/models/foo:embedContent).
|
||||
// bunrouter can't distinguish two routes with the same :param prefix, so a single wildcard
|
||||
// captures the full "model:action" segment and dispatches internally.
|
||||
@@ -58,8 +62,15 @@ func New(
|
||||
|
||||
// Extra-map routes: /map/:mapping/v1/... uses the adapter configured under extra_maps[mapping].
|
||||
authed.POST("/map/:mapping/v1/embeddings", h.openAIEmbeddingsMapped)
|
||||
authed.GET("/map/:mapping/v1/embeddings", h.openAIEmbeddingsGetMapped)
|
||||
authed.POST("/map/:mapping/v1/models/*modelaction", h.googleDispatchMapped)
|
||||
|
||||
// Generic OpenAI-compatible passthrough — forwarded verbatim to the resolved
|
||||
// target with no vecna-specific processing (no dimension adaptation). The
|
||||
// target is chosen the same way as embeddings: the body's "model" field must
|
||||
// name a configured forward target, falling back to forward.default.
|
||||
registerPassthroughRoutes(authed, h)
|
||||
|
||||
// Metrics — only when enabled.
|
||||
// If metrics.api_key is set, routes are guarded by that key (on the authed group).
|
||||
// If metrics.api_key is blank, routes are public — no auth headers checked at all.
|
||||
@@ -85,7 +96,65 @@ func New(
|
||||
return router, nil
|
||||
}
|
||||
|
||||
// authMiddleware rejects requests without a valid Bearer token when api_keys is configured.
|
||||
// recoverMiddleware catches panics from any handler or middleware below it.
|
||||
// Without this, an unrecovered panic kills the request goroutine mid-response
|
||||
// and the client sees a raw connection reset instead of an error; with it,
|
||||
// the client gets a normal 500 JSON error (or, if the panic happened after
|
||||
// the response was already committed — e.g. mid-stream — the connection is
|
||||
// still cut, but the panic is at least logged instead of silently crashing).
|
||||
func recoverMiddleware(logger *zap.Logger) bunrouter.MiddlewareFunc {
|
||||
return func(next bunrouter.HandlerFunc) bunrouter.HandlerFunc {
|
||||
return func(w http.ResponseWriter, req bunrouter.Request) (err error) {
|
||||
rw := &recoverWriter{ResponseWriter: w}
|
||||
defer func() {
|
||||
r := recover()
|
||||
if r == nil {
|
||||
return
|
||||
}
|
||||
logger.Error("panic recovered",
|
||||
zap.Any("recover", r),
|
||||
zap.String("method", req.Method),
|
||||
zap.String("path", req.URL.Path),
|
||||
zap.Stack("stack"),
|
||||
)
|
||||
if rw.wroteHeader {
|
||||
err = fmt.Errorf("panic after response started: %v", r)
|
||||
return
|
||||
}
|
||||
err = writeError(rw, http.StatusInternalServerError, "internal server error")
|
||||
}()
|
||||
return next(rw, req)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// recoverWriter tracks whether a response has already been committed, so
|
||||
// recoverMiddleware knows whether it's still safe to write an error body.
|
||||
type recoverWriter struct {
|
||||
http.ResponseWriter
|
||||
wroteHeader bool
|
||||
}
|
||||
|
||||
func (rw *recoverWriter) WriteHeader(code int) {
|
||||
rw.wroteHeader = true
|
||||
rw.ResponseWriter.WriteHeader(code)
|
||||
}
|
||||
|
||||
func (rw *recoverWriter) Write(b []byte) (int, error) {
|
||||
rw.wroteHeader = true
|
||||
return rw.ResponseWriter.Write(b)
|
||||
}
|
||||
|
||||
func (rw *recoverWriter) Flush() {
|
||||
if f, ok := rw.ResponseWriter.(http.Flusher); ok {
|
||||
f.Flush()
|
||||
}
|
||||
}
|
||||
|
||||
// authMiddleware rejects requests without a valid API key when api_keys is
|
||||
// configured. The key may be supplied as a Bearer token (OpenAI-style), or
|
||||
// via the X-Api-Key or Api-Key headers (Anthropic/Azure OpenAI-style), so
|
||||
// clients built against any of those conventions work unmodified.
|
||||
func authMiddleware(apiKeys []string) bunrouter.MiddlewareFunc {
|
||||
if len(apiKeys) == 0 {
|
||||
return func(next bunrouter.HandlerFunc) bunrouter.HandlerFunc { return next }
|
||||
@@ -96,16 +165,27 @@ func authMiddleware(apiKeys []string) bunrouter.MiddlewareFunc {
|
||||
}
|
||||
return func(next bunrouter.HandlerFunc) bunrouter.HandlerFunc {
|
||||
return func(w http.ResponseWriter, req bunrouter.Request) error {
|
||||
token := strings.TrimPrefix(req.Header.Get("Authorization"), "Bearer ")
|
||||
if _, ok := keySet[token]; !ok {
|
||||
http.Error(w, "unauthorized", http.StatusUnauthorized)
|
||||
return nil
|
||||
token := requestAPIKey(req.Request)
|
||||
if _, ok := keySet[token]; !ok || token == "" {
|
||||
return writeError(w, http.StatusUnauthorized, "unauthorized")
|
||||
}
|
||||
return next(w, req)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// requestAPIKey extracts the client-supplied API key, checking (in order)
|
||||
// the Authorization Bearer scheme, X-Api-Key, and Api-Key headers.
|
||||
func requestAPIKey(req *http.Request) string {
|
||||
if auth := req.Header.Get("Authorization"); auth != "" {
|
||||
return strings.TrimPrefix(auth, "Bearer ")
|
||||
}
|
||||
if key := req.Header.Get("X-Api-Key"); key != "" {
|
||||
return key
|
||||
}
|
||||
return req.Header.Get("Api-Key")
|
||||
}
|
||||
|
||||
// traceMiddleware injects a *RequestTrace into every request context.
|
||||
func traceMiddleware() bunrouter.MiddlewareFunc {
|
||||
return func(next bunrouter.HandlerFunc) bunrouter.HandlerFunc {
|
||||
@@ -161,25 +241,22 @@ func loggingMiddleware(logger *zap.Logger) bunrouter.MiddlewareFunc {
|
||||
}
|
||||
}
|
||||
|
||||
// metricsKeyMiddleware guards a bunrouter.HandlerFunc with a dedicated Bearer token.
|
||||
// metricsKeyMiddleware guards a bunrouter.HandlerFunc with a dedicated API key
|
||||
// (Bearer, X-Api-Key, or Api-Key — see requestAPIKey).
|
||||
func metricsKeyMiddleware(apiKey string, h bunrouter.HandlerFunc) bunrouter.HandlerFunc {
|
||||
return func(w http.ResponseWriter, req bunrouter.Request) error {
|
||||
token := strings.TrimPrefix(req.Header.Get("Authorization"), "Bearer ")
|
||||
if token != apiKey {
|
||||
http.Error(w, "unauthorized", http.StatusUnauthorized)
|
||||
return nil
|
||||
if requestAPIKey(req.Request) != apiKey {
|
||||
return writeError(w, http.StatusUnauthorized, "unauthorized")
|
||||
}
|
||||
return h(w, req)
|
||||
}
|
||||
}
|
||||
|
||||
// metricsAuthHandler wraps a standard http.Handler with Bearer token auth.
|
||||
// metricsAuthHandler wraps a standard http.Handler with the same API key auth.
|
||||
func metricsAuthHandler(apiKey string, h http.Handler) bunrouter.HandlerFunc {
|
||||
return func(w http.ResponseWriter, req bunrouter.Request) error {
|
||||
token := strings.TrimPrefix(req.Header.Get("Authorization"), "Bearer ")
|
||||
if token != apiKey {
|
||||
http.Error(w, "unauthorized", http.StatusUnauthorized)
|
||||
return nil
|
||||
if requestAPIKey(req.Request) != apiKey {
|
||||
return writeError(w, http.StatusUnauthorized, "unauthorized")
|
||||
}
|
||||
h.ServeHTTP(w, req.Request)
|
||||
return nil
|
||||
@@ -196,3 +273,9 @@ func (sw *statusWriter) WriteHeader(code int) {
|
||||
sw.status = code
|
||||
sw.ResponseWriter.WriteHeader(code)
|
||||
}
|
||||
|
||||
func (sw *statusWriter) Flush() {
|
||||
if f, ok := sw.ResponseWriter.(http.Flusher); ok {
|
||||
f.Flush()
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,7 +1,11 @@
|
||||
openapi: "3.1.0"
|
||||
info:
|
||||
title: vecna Embedding Adapter
|
||||
description: Proxies text to a backing embedding model and adapts the result vectors between dimensions.
|
||||
description: >
|
||||
Proxies text to a backing embedding model and adapts the result vectors
|
||||
between dimensions. Also forwards the rest of the OpenAI-compatible API
|
||||
surface (chat, completions, models, audio, images, moderations, rerank)
|
||||
verbatim to the resolved target, with no adaptation applied.
|
||||
version: "1.0.0"
|
||||
|
||||
servers:
|
||||
@@ -21,7 +25,14 @@ components:
|
||||
type: object
|
||||
properties:
|
||||
error:
|
||||
type: string
|
||||
type: object
|
||||
properties:
|
||||
message:
|
||||
type: string
|
||||
type:
|
||||
type: string
|
||||
code:
|
||||
type: integer
|
||||
|
||||
OpenAIEmbedRequest:
|
||||
type: object
|
||||
@@ -169,6 +180,40 @@ paths:
|
||||
"502":
|
||||
description: Backing model error
|
||||
|
||||
get:
|
||||
summary: OpenAI-compatible embeddings (query-param convenience form)
|
||||
operationId: openaiEmbeddingsGet
|
||||
parameters:
|
||||
- name: input
|
||||
in: query
|
||||
required: true
|
||||
description: Repeatable — one or more texts to embed.
|
||||
schema:
|
||||
type: array
|
||||
items:
|
||||
type: string
|
||||
- name: model
|
||||
in: query
|
||||
schema:
|
||||
type: string
|
||||
responses:
|
||||
"200":
|
||||
description: Adapted embeddings
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
$ref: '#/components/schemas/OpenAIEmbedResponse'
|
||||
"400":
|
||||
description: Bad request
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
$ref: '#/components/schemas/Error'
|
||||
"401":
|
||||
description: Unauthorized
|
||||
"502":
|
||||
description: Backing model error
|
||||
|
||||
/v1/models/{model}:embedContent:
|
||||
post:
|
||||
summary: Google-compatible single embedContent
|
||||
@@ -250,3 +295,158 @@ paths:
|
||||
description: Unauthorized
|
||||
"502":
|
||||
description: Backing model error
|
||||
|
||||
# --- Generic OpenAI-compatible passthrough -------------------------------
|
||||
# Forwarded verbatim to the resolved target (chosen by the body's "model"
|
||||
# field, falling back to forward.default). No vecna-specific processing —
|
||||
# request/response shape is whatever the backing model's API defines.
|
||||
# Each also exists under /map/{mapping}/... to force a specific target.
|
||||
/v1/models:
|
||||
get:
|
||||
summary: List models (passthrough)
|
||||
operationId: listModels
|
||||
responses:
|
||||
"200":
|
||||
description: Backend-defined model list
|
||||
"401":
|
||||
description: Unauthorized
|
||||
"502":
|
||||
description: Backing model error
|
||||
|
||||
/v1/models/{model}:
|
||||
get:
|
||||
summary: Retrieve a model (passthrough)
|
||||
operationId: retrieveModel
|
||||
parameters:
|
||||
- name: model
|
||||
in: path
|
||||
required: true
|
||||
schema:
|
||||
type: string
|
||||
responses:
|
||||
"200":
|
||||
description: Backend-defined model object
|
||||
"401":
|
||||
description: Unauthorized
|
||||
"502":
|
||||
description: Backing model error
|
||||
|
||||
/v1/chat/completions:
|
||||
post:
|
||||
summary: Chat completions (passthrough, streaming supported)
|
||||
operationId: chatCompletions
|
||||
responses:
|
||||
"200":
|
||||
description: Backend-defined chat completion (or SSE stream if "stream":true)
|
||||
"401":
|
||||
description: Unauthorized
|
||||
"502":
|
||||
description: Backing model error
|
||||
|
||||
/v1/completions:
|
||||
post:
|
||||
summary: Legacy completions (passthrough, streaming supported)
|
||||
operationId: completions
|
||||
responses:
|
||||
"200":
|
||||
description: Backend-defined completion (or SSE stream if "stream":true)
|
||||
"401":
|
||||
description: Unauthorized
|
||||
"502":
|
||||
description: Backing model error
|
||||
|
||||
/v1/moderations:
|
||||
post:
|
||||
summary: Moderations (passthrough)
|
||||
operationId: moderations
|
||||
responses:
|
||||
"200":
|
||||
description: Backend-defined moderation result
|
||||
"401":
|
||||
description: Unauthorized
|
||||
"502":
|
||||
description: Backing model error
|
||||
|
||||
/v1/images/generations:
|
||||
post:
|
||||
summary: Image generation (passthrough)
|
||||
operationId: imageGenerations
|
||||
responses:
|
||||
"200":
|
||||
description: Backend-defined image result
|
||||
"401":
|
||||
description: Unauthorized
|
||||
"502":
|
||||
description: Backing model error
|
||||
|
||||
/v1/images/edits:
|
||||
post:
|
||||
summary: Image edits (passthrough, multipart/form-data)
|
||||
operationId: imageEdits
|
||||
responses:
|
||||
"200":
|
||||
description: Backend-defined image result
|
||||
"401":
|
||||
description: Unauthorized
|
||||
"502":
|
||||
description: Backing model error
|
||||
|
||||
/v1/images/variations:
|
||||
post:
|
||||
summary: Image variations (passthrough, multipart/form-data)
|
||||
operationId: imageVariations
|
||||
responses:
|
||||
"200":
|
||||
description: Backend-defined image result
|
||||
"401":
|
||||
description: Unauthorized
|
||||
"502":
|
||||
description: Backing model error
|
||||
|
||||
/v1/audio/speech:
|
||||
post:
|
||||
summary: Text-to-speech (passthrough)
|
||||
operationId: audioSpeech
|
||||
responses:
|
||||
"200":
|
||||
description: Backend-defined audio bytes
|
||||
"401":
|
||||
description: Unauthorized
|
||||
"502":
|
||||
description: Backing model error
|
||||
|
||||
/v1/audio/transcriptions:
|
||||
post:
|
||||
summary: Speech-to-text (passthrough, multipart/form-data)
|
||||
operationId: audioTranscriptions
|
||||
responses:
|
||||
"200":
|
||||
description: Backend-defined transcription
|
||||
"401":
|
||||
description: Unauthorized
|
||||
"502":
|
||||
description: Backing model error
|
||||
|
||||
/v1/audio/translations:
|
||||
post:
|
||||
summary: Speech translation (passthrough, multipart/form-data)
|
||||
operationId: audioTranslations
|
||||
responses:
|
||||
"200":
|
||||
description: Backend-defined translation
|
||||
"401":
|
||||
description: Unauthorized
|
||||
"502":
|
||||
description: Backing model error
|
||||
|
||||
/v1/rerank:
|
||||
post:
|
||||
summary: Rerank (passthrough — Cohere/Infinity/vLLM-style)
|
||||
operationId: rerank
|
||||
responses:
|
||||
"200":
|
||||
description: Backend-defined rerank result
|
||||
"401":
|
||||
description: Unauthorized
|
||||
"502":
|
||||
description: Backing model error
|
||||
|
||||
Reference in New Issue
Block a user