mirror of
https://github.com/Warky-Devs/vecna.git
synced 2026-08-07 18:56:06 +00:00
feat(server): add passthrough proxy for OpenAI-compatible endpoints
* implement proxy handler for various OpenAI API routes * add error handling for request body and response streaming * introduce new error response format for API compatibility * add tests for recover middleware to handle panics gracefully
This commit is contained in:
@@ -271,12 +271,17 @@ When unsure, run `vecna test` before and after and compare the reported L2 norm.
|
||||
|
||||
```
|
||||
POST /v1/embeddings
|
||||
GET /v1/embeddings?input=text&input=text2&model=nomic-embed-text
|
||||
Authorization: Bearer <api_key>
|
||||
Content-Type: application/json
|
||||
|
||||
{"input": "text or array of texts", "model": "nomic-embed-text"}
|
||||
```
|
||||
|
||||
Embeddings are the only endpoint that gets dimension adaptation. Everything below is
|
||||
forwarded verbatim (no adaptation) — request/response shape is whatever the backing model's
|
||||
API defines.
|
||||
|
||||
### Google Gemini-compatible
|
||||
|
||||
```
|
||||
@@ -284,14 +289,39 @@ POST /v1/models/{model}:embedContent
|
||||
POST /v1/models/{model}:batchEmbedContents
|
||||
```
|
||||
|
||||
### Generic passthrough
|
||||
|
||||
Target is chosen from the request body's `"model"` field (must name a key under
|
||||
`forward.targets`), falling back to `forward.default`. `"stream": true` requests are
|
||||
relayed as SSE without buffering.
|
||||
|
||||
| Method | Path | Notes |
|
||||
|--------|-------------------------------|-------------------------------|
|
||||
| GET | `/v1/models` | uses `forward.default` |
|
||||
| GET | `/v1/models/{model}` | |
|
||||
| POST | `/v1/chat/completions` | streaming supported |
|
||||
| POST | `/v1/completions` | streaming supported |
|
||||
| POST | `/v1/moderations` | |
|
||||
| POST | `/v1/images/generations` | |
|
||||
| POST | `/v1/images/edits` | multipart/form-data |
|
||||
| POST | `/v1/images/variations` | multipart/form-data |
|
||||
| POST | `/v1/audio/speech` | |
|
||||
| POST | `/v1/audio/transcriptions` | multipart/form-data |
|
||||
| POST | `/v1/audio/translations` | multipart/form-data |
|
||||
| POST | `/v1/rerank` | Cohere/Infinity/vLLM-style |
|
||||
|
||||
### Extra-map routes
|
||||
|
||||
Serve the same backing model with a different adapter per endpoint. The `{mapping}` segment matches a key in `extra_maps`.
|
||||
Serve the same backing model with a different adapter (or forced target) per endpoint.
|
||||
The `{mapping}` segment matches a key in `extra_maps`. Every route above — embeddings,
|
||||
Google, and generic passthrough — also exists under `/map/{mapping}/...`, e.g.:
|
||||
|
||||
```
|
||||
POST /map/{mapping}/v1/embeddings
|
||||
POST /map/{mapping}/v1/models/{model}:embedContent
|
||||
POST /map/{mapping}/v1/models/{model}:batchEmbedContents
|
||||
POST /map/{mapping}/v1/chat/completions
|
||||
GET /map/{mapping}/v1/models
|
||||
```
|
||||
|
||||
All extra-map routes require the same authentication as the standard API routes.
|
||||
|
||||
+48
-1
@@ -14,6 +14,7 @@ import (
|
||||
|
||||
"github.com/Warky-Devs/vecna.git/pkg/config"
|
||||
"github.com/Warky-Devs/vecna.git/pkg/embedclient"
|
||||
"github.com/Warky-Devs/vecna.git/pkg/forwardproxy"
|
||||
"github.com/Warky-Devs/vecna.git/pkg/metrics"
|
||||
"github.com/Warky-Devs/vecna.git/pkg/server"
|
||||
)
|
||||
@@ -58,12 +59,17 @@ func runServe(cmd *cobra.Command, _ []string) error {
|
||||
return fmt.Errorf("build clients: %w", err)
|
||||
}
|
||||
|
||||
forwarders, err := buildForwarders(cfg)
|
||||
if err != nil {
|
||||
return fmt.Errorf("build forwarders: %w", err)
|
||||
}
|
||||
|
||||
var reg *metrics.Registry
|
||||
if cfg.Metrics.Enabled {
|
||||
reg = metrics.New()
|
||||
}
|
||||
|
||||
router, err := server.New(cfg, clients, adp, extraMaps, reg, logger)
|
||||
router, err := server.New(cfg, clients, adp, extraMaps, forwarders, reg, logger)
|
||||
if err != nil {
|
||||
return fmt.Errorf("build router: %w", err)
|
||||
}
|
||||
@@ -156,3 +162,44 @@ func buildClients(cfg *config.Config) (map[string]embedclient.Client, error) {
|
||||
|
||||
return clients, nil
|
||||
}
|
||||
|
||||
// buildForwarders constructs one forwardproxy.Router per named forward target,
|
||||
// used for the generic OpenAI-compatible passthrough endpoints (chat
|
||||
// completions, models list, audio, images, etc.). No client-level timeout is
|
||||
// set here — streaming responses (SSE) are open-ended by design; non-streaming
|
||||
// requests get a per-attempt timeout from RouterConfig.TimeoutSecs.
|
||||
func buildForwarders(cfg *config.Config) (map[string]*forwardproxy.Router, error) {
|
||||
forwarders := make(map[string]*forwardproxy.Router, len(cfg.Forward.Targets))
|
||||
httpClient := &http.Client{}
|
||||
|
||||
for name, target := range cfg.Forward.Targets {
|
||||
if len(target.Endpoints) == 0 {
|
||||
return nil, fmt.Errorf("target %q has no endpoints", name)
|
||||
}
|
||||
|
||||
endpoints := make([]forwardproxy.Endpoint, len(target.Endpoints))
|
||||
for i, ep := range target.Endpoints {
|
||||
apiKey := ep.APIKey
|
||||
if apiKey == "" {
|
||||
apiKey = target.APIKey
|
||||
}
|
||||
endpoints[i] = forwardproxy.Endpoint{URL: ep.URL, APIKey: apiKey, Priority: ep.Priority}
|
||||
}
|
||||
|
||||
routerCfg := forwardproxy.RouterConfig{
|
||||
TargetName: name,
|
||||
TimeoutSecs: target.TimeoutSecs,
|
||||
CooldownSecs: target.CooldownSecs,
|
||||
PriorityDecay: target.PriorityDecay,
|
||||
PriorityRecovery: target.PriorityRecovery,
|
||||
}
|
||||
|
||||
router, err := forwardproxy.NewRouter(endpoints, routerCfg, httpClient)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("build forwarder for target %q: %w", name, err)
|
||||
}
|
||||
forwarders[name] = router
|
||||
}
|
||||
|
||||
return forwarders, nil
|
||||
}
|
||||
|
||||
@@ -0,0 +1,315 @@
|
||||
// Package forwardproxy forwards raw HTTP requests (chat completions, models list,
|
||||
// audio, images, etc.) to a named target's backing endpoints, using the same
|
||||
// priority/cooldown failover algorithm as embedclient.TargetRouter but operating
|
||||
// on raw bytes instead of typed embedding requests — no vecna-specific processing
|
||||
// is applied to the body.
|
||||
package forwardproxy
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Endpoint is a single backing URL for a target.
|
||||
type Endpoint struct {
|
||||
URL string
|
||||
APIKey string
|
||||
Priority int
|
||||
}
|
||||
|
||||
// RouterConfig holds tuning parameters for a Router.
|
||||
type RouterConfig struct {
|
||||
TargetName string
|
||||
TimeoutSecs int
|
||||
CooldownSecs int
|
||||
PriorityDecay int
|
||||
PriorityRecovery int
|
||||
}
|
||||
|
||||
// Result is the outcome of a single forwarded (non-streaming) request.
|
||||
type Result struct {
|
||||
Status int
|
||||
Header http.Header
|
||||
Body []byte
|
||||
Target string // endpoint URL that served the request
|
||||
}
|
||||
|
||||
type slot struct {
|
||||
url string
|
||||
apiKey string
|
||||
initialPriority int
|
||||
|
||||
mu sync.Mutex
|
||||
priority int
|
||||
inflight int
|
||||
successCount int
|
||||
lastFail time.Time
|
||||
}
|
||||
|
||||
// Router forwards raw HTTP requests to the best available endpoint for a target,
|
||||
// retrying against other endpoints on failure.
|
||||
type Router struct {
|
||||
slots []*slot
|
||||
cfg RouterConfig
|
||||
httpClient *http.Client
|
||||
}
|
||||
|
||||
// NewRouter builds a Router from a target's endpoints.
|
||||
func NewRouter(endpoints []Endpoint, cfg RouterConfig, httpClient *http.Client) (*Router, error) {
|
||||
if len(endpoints) == 0 {
|
||||
return nil, fmt.Errorf("forwardproxy.NewRouter: at least one endpoint required")
|
||||
}
|
||||
if httpClient == nil {
|
||||
httpClient = http.DefaultClient
|
||||
}
|
||||
slots := make([]*slot, len(endpoints))
|
||||
for i, ep := range endpoints {
|
||||
slots[i] = &slot{url: ep.URL, apiKey: ep.APIKey, initialPriority: ep.Priority, priority: ep.Priority}
|
||||
}
|
||||
return &Router{slots: slots, cfg: cfg, httpClient: httpClient}, nil
|
||||
}
|
||||
|
||||
// Do forwards method+path+body+headers to the target's base URL + path, retrying
|
||||
// against other endpoints on failure. Buffers the full response body, so it is
|
||||
// not suitable for streaming (SSE) responses — use DoStream for those.
|
||||
func (r *Router) Do(ctx context.Context, method, path string, body []byte, headers http.Header) (Result, error) {
|
||||
tried := make(map[*slot]bool, len(r.slots))
|
||||
var lastErr error
|
||||
|
||||
for range r.slots {
|
||||
if ctx.Err() != nil {
|
||||
return Result{}, ctx.Err()
|
||||
}
|
||||
s := r.pickExcluding(tried)
|
||||
if s == nil {
|
||||
break
|
||||
}
|
||||
tried[s] = true
|
||||
|
||||
timeout := time.Duration(r.cfg.TimeoutSecs) * time.Second
|
||||
reqCtx, cancel := context.WithTimeout(ctx, timeout)
|
||||
res, err := r.doOnce(reqCtx, s, method, path, body, headers)
|
||||
cancel()
|
||||
|
||||
if err != nil {
|
||||
r.onFailure(s)
|
||||
lastErr = fmt.Errorf("forward [%s]: %w", s.url, err)
|
||||
continue
|
||||
}
|
||||
r.onSuccess(s)
|
||||
return res, nil
|
||||
}
|
||||
|
||||
if lastErr != nil {
|
||||
return Result{}, lastErr
|
||||
}
|
||||
return Result{}, fmt.Errorf("forward: no endpoints available")
|
||||
}
|
||||
|
||||
// DoStream forwards a single request to the best available endpoint without
|
||||
// retry, since bytes may already be flushed to the client before a failure is
|
||||
// known. The caller must Close the returned body; doing so records success or
|
||||
// failure against the endpoint's priority score.
|
||||
func (r *Router) DoStream(ctx context.Context, method, path string, body []byte, headers http.Header) (status int, respHeader http.Header, respBody io.ReadCloser, err error) {
|
||||
s := r.pickExcluding(nil)
|
||||
if s == nil {
|
||||
return 0, nil, nil, fmt.Errorf("forward: no endpoints available")
|
||||
}
|
||||
|
||||
httpReq, err := r.buildRequest(ctx, s, method, path, body, headers)
|
||||
if err != nil {
|
||||
return 0, nil, nil, err
|
||||
}
|
||||
|
||||
s.mu.Lock()
|
||||
s.inflight++
|
||||
s.mu.Unlock()
|
||||
|
||||
resp, err := r.httpClient.Do(httpReq) //nolint:bodyclose // closed by the caller via trackedBody.Close
|
||||
if err != nil {
|
||||
s.mu.Lock()
|
||||
s.inflight--
|
||||
s.mu.Unlock()
|
||||
r.onFailure(s)
|
||||
return 0, nil, nil, fmt.Errorf("forward [%s]: %w", s.url, err)
|
||||
}
|
||||
|
||||
return resp.StatusCode, resp.Header, &trackedBody{ReadCloser: resp.Body, onClose: func() {
|
||||
s.mu.Lock()
|
||||
s.inflight--
|
||||
s.mu.Unlock()
|
||||
r.onSuccess(s)
|
||||
}}, nil
|
||||
}
|
||||
|
||||
type trackedBody struct {
|
||||
io.ReadCloser
|
||||
onClose func()
|
||||
once sync.Once
|
||||
}
|
||||
|
||||
func (t *trackedBody) Close() error {
|
||||
err := t.ReadCloser.Close()
|
||||
t.once.Do(t.onClose)
|
||||
return err
|
||||
}
|
||||
|
||||
func (r *Router) doOnce(ctx context.Context, s *slot, method, path string, body []byte, headers http.Header) (Result, error) {
|
||||
s.mu.Lock()
|
||||
s.inflight++
|
||||
s.mu.Unlock()
|
||||
defer func() {
|
||||
s.mu.Lock()
|
||||
s.inflight--
|
||||
s.mu.Unlock()
|
||||
}()
|
||||
|
||||
httpReq, err := r.buildRequest(ctx, s, method, path, body, headers)
|
||||
if err != nil {
|
||||
return Result{}, err
|
||||
}
|
||||
resp, err := r.httpClient.Do(httpReq)
|
||||
if err != nil {
|
||||
return Result{}, err
|
||||
}
|
||||
defer func() { _ = resp.Body.Close() }()
|
||||
|
||||
respBody, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return Result{}, fmt.Errorf("read response: %w", err)
|
||||
}
|
||||
if resp.StatusCode >= 500 {
|
||||
return Result{}, fmt.Errorf("upstream status %d", resp.StatusCode)
|
||||
}
|
||||
return Result{Status: resp.StatusCode, Header: resp.Header, Body: respBody, Target: s.url}, nil
|
||||
}
|
||||
|
||||
func (r *Router) buildRequest(ctx context.Context, s *slot, method, path string, body []byte, headers http.Header) (*http.Request, error) {
|
||||
url := s.url + path
|
||||
var bodyReader io.Reader
|
||||
if body != nil {
|
||||
bodyReader = bytes.NewReader(body)
|
||||
}
|
||||
httpReq, err := http.NewRequestWithContext(ctx, method, url, bodyReader)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("build request: %w", err)
|
||||
}
|
||||
copyHeaders(httpReq.Header, headers, skipOutbound)
|
||||
if s.apiKey != "" {
|
||||
httpReq.Header.Set("Authorization", "Bearer "+s.apiKey)
|
||||
}
|
||||
return httpReq, nil
|
||||
}
|
||||
|
||||
func (r *Router) pickExcluding(excluded map[*slot]bool) *slot {
|
||||
cooldown := time.Duration(r.cfg.CooldownSecs) * time.Second
|
||||
now := time.Now()
|
||||
|
||||
var best *slot
|
||||
bestScore := -1 << 30
|
||||
|
||||
for _, s := range r.slots {
|
||||
if excluded[s] {
|
||||
continue
|
||||
}
|
||||
s.mu.Lock()
|
||||
inCooldown := !s.lastFail.IsZero() && now.Sub(s.lastFail) < cooldown
|
||||
score := s.priority - s.inflight
|
||||
s.mu.Unlock()
|
||||
|
||||
if inCooldown {
|
||||
continue
|
||||
}
|
||||
if best == nil || score > bestScore {
|
||||
best = s
|
||||
bestScore = score
|
||||
}
|
||||
}
|
||||
|
||||
// All non-excluded slots are in cooldown — fall back to the one with the oldest failure.
|
||||
if best == nil {
|
||||
var oldest time.Time
|
||||
for _, s := range r.slots {
|
||||
if excluded[s] {
|
||||
continue
|
||||
}
|
||||
s.mu.Lock()
|
||||
lf := s.lastFail
|
||||
s.mu.Unlock()
|
||||
if best == nil || lf.Before(oldest) {
|
||||
best = s
|
||||
oldest = lf
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return best
|
||||
}
|
||||
|
||||
func (r *Router) onSuccess(s *slot) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
|
||||
s.successCount++
|
||||
if r.cfg.PriorityRecovery > 0 && s.successCount%r.cfg.PriorityRecovery == 0 {
|
||||
if s.priority < s.initialPriority {
|
||||
s.priority++
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (r *Router) onFailure(s *slot) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
|
||||
s.lastFail = time.Now()
|
||||
s.priority -= r.cfg.PriorityDecay
|
||||
if s.priority < 1 {
|
||||
s.priority = 1
|
||||
}
|
||||
}
|
||||
|
||||
// hopByHop headers are never forwarded in either direction.
|
||||
var hopByHop = map[string]bool{
|
||||
"Connection": true,
|
||||
"Proxy-Connection": true,
|
||||
"Keep-Alive": true,
|
||||
"Transfer-Encoding": true,
|
||||
"Te": true,
|
||||
"Trailer": true,
|
||||
"Upgrade": true,
|
||||
"Content-Length": true, // net/http (re)computes this from the body it's given
|
||||
}
|
||||
|
||||
// skipOutbound headers are additionally stripped from client->upstream requests:
|
||||
// these authenticate the client to vecna, not vecna to the upstream, and are
|
||||
// replaced with the endpoint's own api key (as an Authorization Bearer header), if any.
|
||||
var skipOutbound = map[string]bool{
|
||||
"Host": true,
|
||||
"Authorization": true,
|
||||
"X-Api-Key": true,
|
||||
"Api-Key": true,
|
||||
}
|
||||
|
||||
// SkipResponseHeader reports whether a header should not be copied from the
|
||||
// upstream response back to the original client.
|
||||
func SkipResponseHeader(name string) bool {
|
||||
return hopByHop[http.CanonicalHeaderKey(name)]
|
||||
}
|
||||
|
||||
func copyHeaders(dst, src http.Header, extraSkip map[string]bool) {
|
||||
for k, vv := range src {
|
||||
ck := http.CanonicalHeaderKey(k)
|
||||
if hopByHop[ck] || extraSkip[ck] {
|
||||
continue
|
||||
}
|
||||
for _, v := range vv {
|
||||
dst.Add(k, v)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -60,3 +60,9 @@ func (sw *statusWriter) WriteHeader(code int) {
|
||||
sw.status = code
|
||||
sw.ResponseWriter.WriteHeader(code)
|
||||
}
|
||||
|
||||
func (sw *statusWriter) Flush() {
|
||||
if f, ok := sw.ResponseWriter.(http.Flusher); ok {
|
||||
f.Flush()
|
||||
}
|
||||
}
|
||||
|
||||
@@ -23,7 +23,7 @@ func (h *handler) googleDispatch(w http.ResponseWriter, req bunrouter.Request) e
|
||||
func (h *handler) googleDispatchMapped(w http.ResponseWriter, req bunrouter.Request) error {
|
||||
em, err := h.resolveExtraMap(req.Param("mapping"))
|
||||
if err != nil {
|
||||
return writeJSON(w, http.StatusNotFound, map[string]string{"error": err.Error()})
|
||||
return writeError(w, http.StatusNotFound, err.Error())
|
||||
}
|
||||
return h.googleDispatchWithAdapter(w, req, em.Adapter, em.ForwardTarget)
|
||||
}
|
||||
@@ -32,7 +32,7 @@ func (h *handler) googleDispatchWithAdapter(w http.ResponseWriter, req bunrouter
|
||||
modelaction := req.Param("modelaction") // e.g. "text-embedding-foo:embedContent"
|
||||
idx := strings.LastIndex(modelaction, ":")
|
||||
if idx < 0 {
|
||||
return writeJSON(w, http.StatusNotFound, map[string]string{"error": "invalid Google API path"})
|
||||
return writeError(w, http.StatusNotFound, "invalid Google API path")
|
||||
}
|
||||
model := modelaction[:idx]
|
||||
action := modelaction[idx+1:]
|
||||
@@ -46,7 +46,7 @@ func (h *handler) googleDispatchWithAdapter(w http.ResponseWriter, req bunrouter
|
||||
case "batchEmbedContents":
|
||||
return h.googleBatchEmbedContentsWithAdapter(w, req, adp, targetOverride)
|
||||
default:
|
||||
return writeJSON(w, http.StatusNotFound, map[string]string{"error": "unknown Google API method: " + action})
|
||||
return writeError(w, http.StatusNotFound, "unknown Google API method: "+action)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -78,7 +78,7 @@ func (h *handler) googleEmbedContentWithAdapter(w http.ResponseWriter, req bunro
|
||||
|
||||
var body googleEmbedContentRequest
|
||||
if err := json.NewDecoder(req.Body).Decode(&body); err != nil {
|
||||
return writeJSON(w, http.StatusBadRequest, map[string]string{"error": "invalid request body"})
|
||||
return writeError(w, http.StatusBadRequest, "invalid request body")
|
||||
}
|
||||
|
||||
texts := make([]string, len(body.Content.Parts))
|
||||
@@ -95,7 +95,7 @@ func (h *handler) googleEmbedContentWithAdapter(w http.ResponseWriter, req bunro
|
||||
embedResp, err := client.Embed(req.Context(), embedclient.Request{Texts: texts, Model: model})
|
||||
trace.ForwardDuration = time.Since(t0)
|
||||
if err != nil {
|
||||
return writeJSON(w, http.StatusBadGateway, map[string]string{"error": err.Error()})
|
||||
return writeError(w, http.StatusBadGateway, err.Error())
|
||||
}
|
||||
trace.ForwardModel = embedResp.Model
|
||||
trace.PromptTokens = embedResp.Usage.PromptTokens
|
||||
@@ -106,7 +106,7 @@ func (h *handler) googleEmbedContentWithAdapter(w http.ResponseWriter, req bunro
|
||||
if len(embedResp.Embeddings) > 0 {
|
||||
adapted, err = adp.Adapt(embedResp.Embeddings[0])
|
||||
if err != nil {
|
||||
return writeJSON(w, http.StatusInternalServerError, map[string]string{"error": err.Error()})
|
||||
return writeError(w, http.StatusInternalServerError, err.Error())
|
||||
}
|
||||
}
|
||||
trace.TranslateDuration = time.Since(t1)
|
||||
@@ -133,7 +133,7 @@ func (h *handler) googleBatchEmbedContentsWithAdapter(w http.ResponseWriter, req
|
||||
|
||||
var body googleBatchRequest
|
||||
if err := json.NewDecoder(req.Body).Decode(&body); err != nil {
|
||||
return writeJSON(w, http.StatusBadRequest, map[string]string{"error": "invalid request body"})
|
||||
return writeError(w, http.StatusBadRequest, "invalid request body")
|
||||
}
|
||||
|
||||
var texts []string
|
||||
@@ -152,7 +152,7 @@ func (h *handler) googleBatchEmbedContentsWithAdapter(w http.ResponseWriter, req
|
||||
embedResp, err := client.Embed(req.Context(), embedclient.Request{Texts: texts, Model: model})
|
||||
trace.ForwardDuration = time.Since(t0)
|
||||
if err != nil {
|
||||
return writeJSON(w, http.StatusBadGateway, map[string]string{"error": err.Error()})
|
||||
return writeError(w, http.StatusBadGateway, err.Error())
|
||||
}
|
||||
trace.ForwardModel = embedResp.Model
|
||||
trace.PromptTokens = embedResp.Usage.PromptTokens
|
||||
@@ -163,7 +163,7 @@ func (h *handler) googleBatchEmbedContentsWithAdapter(w http.ResponseWriter, req
|
||||
for i, vec := range embedResp.Embeddings {
|
||||
adapted, adaptErr := adp.Adapt(vec)
|
||||
if adaptErr != nil {
|
||||
return writeJSON(w, http.StatusInternalServerError, map[string]string{"error": adaptErr.Error()})
|
||||
return writeError(w, http.StatusInternalServerError, adaptErr.Error())
|
||||
}
|
||||
result[i] = googleEmbeddingValues{Values: adapted}
|
||||
}
|
||||
|
||||
@@ -12,6 +12,7 @@ import (
|
||||
"github.com/Warky-Devs/vecna.git/pkg/adapter"
|
||||
"github.com/Warky-Devs/vecna.git/pkg/config"
|
||||
"github.com/Warky-Devs/vecna.git/pkg/embedclient"
|
||||
"github.com/Warky-Devs/vecna.git/pkg/forwardproxy"
|
||||
)
|
||||
|
||||
// ExtraMap pairs a dimension adapter with an optional forward-target override.
|
||||
@@ -26,6 +27,7 @@ type handler struct {
|
||||
clients map[string]embedclient.Client
|
||||
adapter adapter.Adapter
|
||||
extraMaps map[string]ExtraMap
|
||||
forwarders map[string]*forwardproxy.Router
|
||||
logger *zap.Logger
|
||||
}
|
||||
|
||||
@@ -66,6 +68,33 @@ func (h *handler) resolveClientOverride(targetOverride, model string) (embedclie
|
||||
return c, targetOverride, firstEndpointURL(h.cfg, targetOverride)
|
||||
}
|
||||
|
||||
// resolveForwarder selects the raw forwarder for the given model, falling back
|
||||
// to forward.default when the model doesn't name a configured target directly.
|
||||
func (h *handler) resolveForwarder(model string) (*forwardproxy.Router, string, error) {
|
||||
if f, ok := h.forwarders[model]; ok {
|
||||
return f, model, nil
|
||||
}
|
||||
name := h.cfg.Forward.Default
|
||||
f, ok := h.forwarders[name]
|
||||
if !ok {
|
||||
return nil, name, fmt.Errorf("no forward target configured for model %q and no default", model)
|
||||
}
|
||||
return f, name, nil
|
||||
}
|
||||
|
||||
// resolveForwarderOverride selects the forwarder for targetOverride when set,
|
||||
// otherwise falls back to model-based resolution.
|
||||
func (h *handler) resolveForwarderOverride(targetOverride, model string) (*forwardproxy.Router, string, error) {
|
||||
if targetOverride == "" {
|
||||
return h.resolveForwarder(model)
|
||||
}
|
||||
f, ok := h.forwarders[targetOverride]
|
||||
if !ok {
|
||||
return nil, targetOverride, fmt.Errorf("extra_map forward_target %q not configured", targetOverride)
|
||||
}
|
||||
return f, targetOverride, nil
|
||||
}
|
||||
|
||||
func firstEndpointURL(cfg *config.Config, targetName string) string {
|
||||
t, ok := cfg.Forward.Targets[targetName]
|
||||
if !ok || len(t.Endpoints) == 0 {
|
||||
@@ -84,6 +113,45 @@ func writeJSON(w http.ResponseWriter, status int, v interface{}) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// apiError is the OpenAI-compatible error envelope, so existing OpenAI SDKs
|
||||
// (which read err.error.message) surface vecna's own errors correctly instead
|
||||
// of failing to parse a flat {"error": "..."} string.
|
||||
type apiError struct {
|
||||
Message string `json:"message"`
|
||||
Type string `json:"type"`
|
||||
Code int `json:"code"`
|
||||
}
|
||||
|
||||
type apiErrorEnvelope struct {
|
||||
Error apiError `json:"error"`
|
||||
}
|
||||
|
||||
// writeError writes an OpenAI-compatible error response.
|
||||
func writeError(w http.ResponseWriter, status int, message string) error {
|
||||
return writeJSON(w, status, apiErrorEnvelope{Error: apiError{
|
||||
Message: message,
|
||||
Type: errorType(status),
|
||||
Code: status,
|
||||
}})
|
||||
}
|
||||
|
||||
// errorType maps an HTTP status to an OpenAI-style error type string.
|
||||
func errorType(status int) string {
|
||||
switch status {
|
||||
case http.StatusUnauthorized:
|
||||
return "authentication_error"
|
||||
case http.StatusBadRequest, http.StatusNotFound:
|
||||
return "invalid_request_error"
|
||||
case http.StatusBadGateway:
|
||||
return "upstream_error"
|
||||
default:
|
||||
if status >= 500 {
|
||||
return "internal_error"
|
||||
}
|
||||
return "api_error"
|
||||
}
|
||||
}
|
||||
|
||||
// writeTraceHeaders writes X-Vecna-* timing headers from the RequestTrace.
|
||||
func writeTraceHeaders(w http.ResponseWriter, t *RequestTrace) {
|
||||
total := time.Since(t.Start)
|
||||
|
||||
+34
-7
@@ -42,32 +42,59 @@ func (h *handler) openAIEmbeddings(w http.ResponseWriter, req bunrouter.Request)
|
||||
func (h *handler) openAIEmbeddingsMapped(w http.ResponseWriter, req bunrouter.Request) error {
|
||||
em, err := h.resolveExtraMap(req.Param("mapping"))
|
||||
if err != nil {
|
||||
return writeJSON(w, http.StatusNotFound, map[string]string{"error": err.Error()})
|
||||
return writeError(w, http.StatusNotFound, err.Error())
|
||||
}
|
||||
return h.openAIEmbeddingsWithAdapter(w, req, em.Adapter, em.ForwardTarget)
|
||||
}
|
||||
|
||||
// openAIEmbeddingsGet is a GET convenience variant (e.g. for browser/curl testing):
|
||||
// ?input=foo&input=bar&model=name in place of a JSON body.
|
||||
func (h *handler) openAIEmbeddingsGet(w http.ResponseWriter, req bunrouter.Request) error {
|
||||
return h.openAIEmbeddingsGetWithAdapter(w, req, h.adapter, "")
|
||||
}
|
||||
|
||||
func (h *handler) openAIEmbeddingsGetMapped(w http.ResponseWriter, req bunrouter.Request) error {
|
||||
em, err := h.resolveExtraMap(req.Param("mapping"))
|
||||
if err != nil {
|
||||
return writeError(w, http.StatusNotFound, err.Error())
|
||||
}
|
||||
return h.openAIEmbeddingsGetWithAdapter(w, req, em.Adapter, em.ForwardTarget)
|
||||
}
|
||||
|
||||
func (h *handler) openAIEmbeddingsGetWithAdapter(w http.ResponseWriter, req bunrouter.Request, adp adapter.Adapter, targetOverride string) error {
|
||||
q := req.URL.Query()
|
||||
texts := q["input"]
|
||||
if len(texts) == 0 {
|
||||
return writeError(w, http.StatusBadRequest, "query parameter \"input\" is required")
|
||||
}
|
||||
return h.processEmbeddings(w, req, adp, targetOverride, texts, q.Get("model"))
|
||||
}
|
||||
|
||||
func (h *handler) openAIEmbeddingsWithAdapter(w http.ResponseWriter, req bunrouter.Request, adp adapter.Adapter, targetOverride string) error {
|
||||
var body openAIEmbedRequest
|
||||
if err := json.NewDecoder(req.Body).Decode(&body); err != nil {
|
||||
return writeJSON(w, http.StatusBadRequest, map[string]string{"error": "invalid request body"})
|
||||
return writeError(w, http.StatusBadRequest, "invalid request body")
|
||||
}
|
||||
|
||||
texts, err := toStringSlice(body.Input)
|
||||
if err != nil {
|
||||
return writeJSON(w, http.StatusBadRequest, map[string]string{"error": err.Error()})
|
||||
return writeError(w, http.StatusBadRequest, err.Error())
|
||||
}
|
||||
|
||||
client, targetName, targetURL := h.resolveClientOverride(targetOverride, body.Model)
|
||||
return h.processEmbeddings(w, req, adp, targetOverride, texts, body.Model)
|
||||
}
|
||||
|
||||
func (h *handler) processEmbeddings(w http.ResponseWriter, req bunrouter.Request, adp adapter.Adapter, targetOverride string, texts []string, model string) error {
|
||||
client, targetName, targetURL := h.resolveClientOverride(targetOverride, model)
|
||||
trace := TraceFromContext(req.Context())
|
||||
trace.ForwardTarget = targetName
|
||||
trace.ForwardURL = targetURL
|
||||
|
||||
t0 := time.Now()
|
||||
embedResp, err := client.Embed(req.Context(), embedclient.Request{Texts: texts, Model: body.Model})
|
||||
embedResp, err := client.Embed(req.Context(), embedclient.Request{Texts: texts, Model: model})
|
||||
trace.ForwardDuration = time.Since(t0)
|
||||
if err != nil {
|
||||
return writeJSON(w, http.StatusBadGateway, map[string]string{"error": err.Error()})
|
||||
return writeError(w, http.StatusBadGateway, err.Error())
|
||||
}
|
||||
trace.ForwardModel = embedResp.Model
|
||||
trace.PromptTokens = embedResp.Usage.PromptTokens
|
||||
@@ -78,7 +105,7 @@ func (h *handler) openAIEmbeddingsWithAdapter(w http.ResponseWriter, req bunrout
|
||||
for i, vec := range embedResp.Embeddings {
|
||||
adapted, adaptErr := adp.Adapt(vec)
|
||||
if adaptErr != nil {
|
||||
return writeJSON(w, http.StatusInternalServerError, map[string]string{"error": adaptErr.Error()})
|
||||
return writeError(w, http.StatusInternalServerError, adaptErr.Error())
|
||||
}
|
||||
data[i] = openAIEmbedDatum{Object: "embedding", Embedding: adapted, Index: i}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,182 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"io"
|
||||
"net/http"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/uptrace/bunrouter"
|
||||
|
||||
"github.com/Warky-Devs/vecna.git/pkg/forwardproxy"
|
||||
)
|
||||
|
||||
// passthroughRoute describes one OpenAI-compatible endpoint forwarded verbatim.
|
||||
type passthroughRoute struct {
|
||||
method string
|
||||
path string // route pattern registered with bunrouter (":model" etc.)
|
||||
}
|
||||
|
||||
// passthroughRoutes lists the OpenAI-compatible endpoints vecna forwards without
|
||||
// any body processing, beyond /v1/embeddings (which gets dimension adaptation).
|
||||
var passthroughRoutes = []passthroughRoute{
|
||||
{http.MethodGet, "/v1/models"},
|
||||
{http.MethodGet, "/v1/models/:model"},
|
||||
{http.MethodPost, "/v1/chat/completions"},
|
||||
{http.MethodPost, "/v1/completions"},
|
||||
{http.MethodPost, "/v1/moderations"},
|
||||
{http.MethodPost, "/v1/images/generations"},
|
||||
{http.MethodPost, "/v1/images/edits"},
|
||||
{http.MethodPost, "/v1/images/variations"},
|
||||
{http.MethodPost, "/v1/audio/speech"},
|
||||
{http.MethodPost, "/v1/audio/transcriptions"},
|
||||
{http.MethodPost, "/v1/audio/translations"},
|
||||
{http.MethodPost, "/v1/rerank"},
|
||||
}
|
||||
|
||||
// registerPassthroughRoutes wires up both the unmapped and /map/:mapping variants
|
||||
// of every entry in passthroughRoutes.
|
||||
func registerPassthroughRoutes(authed *bunrouter.Group, h *handler) {
|
||||
for _, r := range passthroughRoutes {
|
||||
upstreamPath := strings.Replace(r.path, ":model", "{model}", 1)
|
||||
authed.Handle(r.method, r.path, h.proxyHandler(r.method, upstreamPath))
|
||||
authed.Handle(r.method, "/map/:mapping"+r.path, h.proxyHandlerMapped(r.method, upstreamPath))
|
||||
}
|
||||
}
|
||||
|
||||
// proxyHandler returns a handler that forwards method/path verbatim to the
|
||||
// target resolved from the request body's "model" field (or forward.default).
|
||||
// No vecna-specific processing (adaptation, etc.) is applied to the body.
|
||||
func (h *handler) proxyHandler(method, path string) bunrouter.HandlerFunc {
|
||||
return func(w http.ResponseWriter, req bunrouter.Request) error {
|
||||
return h.doProxy(w, req, method, path, "")
|
||||
}
|
||||
}
|
||||
|
||||
// proxyHandlerMapped is the /map/:mapping variant: the target is always the
|
||||
// extra_map's forward_target, regardless of the body's "model" field.
|
||||
func (h *handler) proxyHandlerMapped(method, path string) bunrouter.HandlerFunc {
|
||||
return func(w http.ResponseWriter, req bunrouter.Request) error {
|
||||
em, err := h.resolveExtraMap(req.Param("mapping"))
|
||||
if err != nil {
|
||||
return writeError(w, http.StatusNotFound, err.Error())
|
||||
}
|
||||
return h.doProxy(w, req, method, path, em.ForwardTarget)
|
||||
}
|
||||
}
|
||||
|
||||
func (h *handler) doProxy(w http.ResponseWriter, req bunrouter.Request, method, pathTemplate, targetOverride string) error {
|
||||
model := req.Param("model")
|
||||
upstreamPath := strings.Replace(pathTemplate, "{model}", model, 1)
|
||||
|
||||
var body []byte
|
||||
if method != http.MethodGet && method != http.MethodDelete {
|
||||
b, err := io.ReadAll(req.Body)
|
||||
if err != nil {
|
||||
return writeError(w, http.StatusBadRequest, "invalid request body")
|
||||
}
|
||||
body = b
|
||||
if model == "" && strings.HasPrefix(req.Header.Get("Content-Type"), "application/json") {
|
||||
model = modelPeek(body)
|
||||
}
|
||||
}
|
||||
|
||||
fwd, targetName, err := h.resolveForwarderOverride(targetOverride, model)
|
||||
if err != nil {
|
||||
return writeError(w, http.StatusBadGateway, err.Error())
|
||||
}
|
||||
|
||||
trace := TraceFromContext(req.Context())
|
||||
trace.ForwardTarget = targetName
|
||||
|
||||
if isStreaming(body) {
|
||||
return h.streamProxy(w, req, fwd, method, upstreamPath, body, trace)
|
||||
}
|
||||
|
||||
t0 := time.Now()
|
||||
res, err := fwd.Do(req.Context(), method, upstreamPath, body, req.Header)
|
||||
trace.ForwardDuration = time.Since(t0)
|
||||
if err != nil {
|
||||
return writeError(w, http.StatusBadGateway, err.Error())
|
||||
}
|
||||
trace.ForwardURL = res.Target
|
||||
|
||||
writeTraceHeaders(w, trace)
|
||||
writeUpstreamResult(w, res)
|
||||
return nil
|
||||
}
|
||||
|
||||
// streamProxy relays a single-attempt streaming response (e.g. SSE chat
|
||||
// completions) directly to the client as bytes arrive.
|
||||
func (h *handler) streamProxy(w http.ResponseWriter, req bunrouter.Request, fwd *forwardproxy.Router, method, path string, body []byte, trace *RequestTrace) error {
|
||||
t0 := time.Now()
|
||||
status, header, respBody, err := fwd.DoStream(req.Context(), method, path, body, req.Header)
|
||||
trace.ForwardDuration = time.Since(t0)
|
||||
if err != nil {
|
||||
return writeError(w, http.StatusBadGateway, err.Error())
|
||||
}
|
||||
defer func() { _ = respBody.Close() }()
|
||||
|
||||
dst := w.Header()
|
||||
for k, vv := range header {
|
||||
if forwardproxy.SkipResponseHeader(k) {
|
||||
continue
|
||||
}
|
||||
for _, v := range vv {
|
||||
dst.Add(k, v)
|
||||
}
|
||||
}
|
||||
w.WriteHeader(status)
|
||||
|
||||
flusher, _ := w.(http.Flusher)
|
||||
buf := make([]byte, 4096)
|
||||
for {
|
||||
n, rerr := respBody.Read(buf)
|
||||
if n > 0 {
|
||||
if _, werr := w.Write(buf[:n]); werr != nil {
|
||||
return nil
|
||||
}
|
||||
if flusher != nil {
|
||||
flusher.Flush()
|
||||
}
|
||||
}
|
||||
if rerr != nil {
|
||||
break
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func writeUpstreamResult(w http.ResponseWriter, res forwardproxy.Result) {
|
||||
dst := w.Header()
|
||||
for k, vv := range res.Header {
|
||||
if forwardproxy.SkipResponseHeader(k) {
|
||||
continue
|
||||
}
|
||||
for _, v := range vv {
|
||||
dst.Add(k, v)
|
||||
}
|
||||
}
|
||||
w.WriteHeader(res.Status)
|
||||
_, _ = w.Write(res.Body)
|
||||
}
|
||||
|
||||
// modelPeek extracts the "model" field from a JSON body without full decoding,
|
||||
// used to select a forward target the same way embeddings does.
|
||||
func modelPeek(body []byte) string {
|
||||
var v struct {
|
||||
Model string `json:"model"`
|
||||
}
|
||||
_ = json.Unmarshal(body, &v)
|
||||
return v.Model
|
||||
}
|
||||
|
||||
// isStreaming reports whether a JSON request body sets "stream": true.
|
||||
func isStreaming(body []byte) bool {
|
||||
var v struct {
|
||||
Stream bool `json:"stream"`
|
||||
}
|
||||
_ = json.Unmarshal(body, &v)
|
||||
return v.Stream
|
||||
}
|
||||
@@ -0,0 +1,88 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
|
||||
"github.com/uptrace/bunrouter"
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
||||
// TestRecoverMiddlewarePanicBeforeResponse verifies a panic in a handler that
|
||||
// hasn't written anything yet is converted into a 500 JSON error instead of
|
||||
// crashing the request goroutine (which would otherwise surface to the client
|
||||
// as a connection reset).
|
||||
func TestRecoverMiddlewarePanicBeforeResponse(t *testing.T) {
|
||||
mw := recoverMiddleware(zap.NewNop())
|
||||
handler := mw(func(w http.ResponseWriter, req bunrouter.Request) error {
|
||||
panic("boom")
|
||||
})
|
||||
|
||||
req := httptest.NewRequestWithContext(context.Background(), http.MethodGet, "/v1/models", nil)
|
||||
w := httptest.NewRecorder()
|
||||
|
||||
err := handler(w, bunrouter.Request{Request: req})
|
||||
if err != nil {
|
||||
t.Fatalf("expected recoverMiddleware to swallow the panic and return nil, got %v", err)
|
||||
}
|
||||
if w.Code != http.StatusInternalServerError {
|
||||
t.Fatalf("expected status 500, got %d", w.Code)
|
||||
}
|
||||
|
||||
var body apiErrorEnvelope
|
||||
if err := json.NewDecoder(w.Body).Decode(&body); err != nil {
|
||||
t.Fatalf("expected a valid JSON error body, got decode error: %v (body=%q)", err, w.Body.String())
|
||||
}
|
||||
if body.Error.Type != "internal_error" {
|
||||
t.Fatalf("expected error type internal_error, got %q", body.Error.Type)
|
||||
}
|
||||
}
|
||||
|
||||
// TestRecoverMiddlewarePanicAfterResponse verifies a panic that happens after
|
||||
// the response has already been committed (e.g. mid-stream) doesn't attempt a
|
||||
// second WriteHeader — it just surfaces as an error for the logger.
|
||||
func TestRecoverMiddlewarePanicAfterResponse(t *testing.T) {
|
||||
mw := recoverMiddleware(zap.NewNop())
|
||||
handler := mw(func(w http.ResponseWriter, req bunrouter.Request) error {
|
||||
w.WriteHeader(http.StatusOK)
|
||||
_, _ = w.Write([]byte("partial"))
|
||||
panic("boom mid-stream")
|
||||
})
|
||||
|
||||
req := httptest.NewRequestWithContext(context.Background(), http.MethodGet, "/v1/chat/completions", nil)
|
||||
w := httptest.NewRecorder()
|
||||
|
||||
err := handler(w, bunrouter.Request{Request: req})
|
||||
if err == nil {
|
||||
t.Fatal("expected an error to be returned once the response was already committed")
|
||||
}
|
||||
if w.Code != http.StatusOK {
|
||||
t.Fatalf("expected the original 200 to stand, got %d", w.Code)
|
||||
}
|
||||
if w.Body.String() != "partial" {
|
||||
t.Fatalf("expected body to be left as-is, got %q", w.Body.String())
|
||||
}
|
||||
}
|
||||
|
||||
// TestRecoverMiddlewareNoPanic verifies the happy path is unaffected.
|
||||
func TestRecoverMiddlewareNoPanic(t *testing.T) {
|
||||
mw := recoverMiddleware(zap.NewNop())
|
||||
handler := mw(func(w http.ResponseWriter, req bunrouter.Request) error {
|
||||
w.WriteHeader(http.StatusOK)
|
||||
_, _ = w.Write([]byte("ok"))
|
||||
return nil
|
||||
})
|
||||
|
||||
req := httptest.NewRequestWithContext(context.Background(), http.MethodGet, "/v1/models", nil)
|
||||
w := httptest.NewRecorder()
|
||||
|
||||
if err := handler(w, bunrouter.Request{Request: req}); err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
if w.Code != http.StatusOK || w.Body.String() != "ok" {
|
||||
t.Fatalf("unexpected response: status=%d body=%q", w.Code, w.Body.String())
|
||||
}
|
||||
}
|
||||
+99
-16
@@ -13,6 +13,7 @@ import (
|
||||
"github.com/Warky-Devs/vecna.git/pkg/adapter"
|
||||
"github.com/Warky-Devs/vecna.git/pkg/config"
|
||||
"github.com/Warky-Devs/vecna.git/pkg/embedclient"
|
||||
"github.com/Warky-Devs/vecna.git/pkg/forwardproxy"
|
||||
"github.com/Warky-Devs/vecna.git/pkg/metrics"
|
||||
"github.com/Warky-Devs/vecna.git/pkg/server/spec"
|
||||
)
|
||||
@@ -24,6 +25,7 @@ func New(
|
||||
clients map[string]embedclient.Client,
|
||||
adp adapter.Adapter,
|
||||
extraMaps map[string]ExtraMap,
|
||||
forwarders map[string]*forwardproxy.Router,
|
||||
reg *metrics.Registry,
|
||||
logger *zap.Logger,
|
||||
) (router *bunrouter.Router, err error) {
|
||||
@@ -35,12 +37,13 @@ func New(
|
||||
}()
|
||||
|
||||
router = bunrouter.New(
|
||||
bunrouter.WithMiddleware(recoverMiddleware(logger)),
|
||||
bunrouter.WithMiddleware(traceMiddleware()),
|
||||
bunrouter.WithMiddleware(metricsMiddleware(reg, adp)),
|
||||
bunrouter.WithMiddleware(loggingMiddleware(logger)),
|
||||
)
|
||||
|
||||
h := &handler{cfg: cfg, clients: clients, adapter: adp, extraMaps: extraMaps, logger: logger}
|
||||
h := &handler{cfg: cfg, clients: clients, adapter: adp, extraMaps: extraMaps, forwarders: forwarders, logger: logger}
|
||||
|
||||
// Public routes — no authentication required.
|
||||
router.GET("/", spec.DocsHandler())
|
||||
@@ -51,6 +54,7 @@ func New(
|
||||
authed := router.NewGroup("", bunrouter.WithMiddleware(authMiddleware(cfg.Server.APIKeys)))
|
||||
|
||||
authed.POST("/v1/embeddings", h.openAIEmbeddings)
|
||||
authed.GET("/v1/embeddings", h.openAIEmbeddingsGet)
|
||||
// Google API uses a literal colon as a method separator (e.g. /v1/models/foo:embedContent).
|
||||
// bunrouter can't distinguish two routes with the same :param prefix, so a single wildcard
|
||||
// captures the full "model:action" segment and dispatches internally.
|
||||
@@ -58,8 +62,15 @@ func New(
|
||||
|
||||
// Extra-map routes: /map/:mapping/v1/... uses the adapter configured under extra_maps[mapping].
|
||||
authed.POST("/map/:mapping/v1/embeddings", h.openAIEmbeddingsMapped)
|
||||
authed.GET("/map/:mapping/v1/embeddings", h.openAIEmbeddingsGetMapped)
|
||||
authed.POST("/map/:mapping/v1/models/*modelaction", h.googleDispatchMapped)
|
||||
|
||||
// Generic OpenAI-compatible passthrough — forwarded verbatim to the resolved
|
||||
// target with no vecna-specific processing (no dimension adaptation). The
|
||||
// target is chosen the same way as embeddings: the body's "model" field must
|
||||
// name a configured forward target, falling back to forward.default.
|
||||
registerPassthroughRoutes(authed, h)
|
||||
|
||||
// Metrics — only when enabled.
|
||||
// If metrics.api_key is set, routes are guarded by that key (on the authed group).
|
||||
// If metrics.api_key is blank, routes are public — no auth headers checked at all.
|
||||
@@ -85,7 +96,65 @@ func New(
|
||||
return router, nil
|
||||
}
|
||||
|
||||
// authMiddleware rejects requests without a valid Bearer token when api_keys is configured.
|
||||
// recoverMiddleware catches panics from any handler or middleware below it.
|
||||
// Without this, an unrecovered panic kills the request goroutine mid-response
|
||||
// and the client sees a raw connection reset instead of an error; with it,
|
||||
// the client gets a normal 500 JSON error (or, if the panic happened after
|
||||
// the response was already committed — e.g. mid-stream — the connection is
|
||||
// still cut, but the panic is at least logged instead of silently crashing).
|
||||
func recoverMiddleware(logger *zap.Logger) bunrouter.MiddlewareFunc {
|
||||
return func(next bunrouter.HandlerFunc) bunrouter.HandlerFunc {
|
||||
return func(w http.ResponseWriter, req bunrouter.Request) (err error) {
|
||||
rw := &recoverWriter{ResponseWriter: w}
|
||||
defer func() {
|
||||
r := recover()
|
||||
if r == nil {
|
||||
return
|
||||
}
|
||||
logger.Error("panic recovered",
|
||||
zap.Any("recover", r),
|
||||
zap.String("method", req.Method),
|
||||
zap.String("path", req.URL.Path),
|
||||
zap.Stack("stack"),
|
||||
)
|
||||
if rw.wroteHeader {
|
||||
err = fmt.Errorf("panic after response started: %v", r)
|
||||
return
|
||||
}
|
||||
err = writeError(rw, http.StatusInternalServerError, "internal server error")
|
||||
}()
|
||||
return next(rw, req)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// recoverWriter tracks whether a response has already been committed, so
|
||||
// recoverMiddleware knows whether it's still safe to write an error body.
|
||||
type recoverWriter struct {
|
||||
http.ResponseWriter
|
||||
wroteHeader bool
|
||||
}
|
||||
|
||||
func (rw *recoverWriter) WriteHeader(code int) {
|
||||
rw.wroteHeader = true
|
||||
rw.ResponseWriter.WriteHeader(code)
|
||||
}
|
||||
|
||||
func (rw *recoverWriter) Write(b []byte) (int, error) {
|
||||
rw.wroteHeader = true
|
||||
return rw.ResponseWriter.Write(b)
|
||||
}
|
||||
|
||||
func (rw *recoverWriter) Flush() {
|
||||
if f, ok := rw.ResponseWriter.(http.Flusher); ok {
|
||||
f.Flush()
|
||||
}
|
||||
}
|
||||
|
||||
// authMiddleware rejects requests without a valid API key when api_keys is
|
||||
// configured. The key may be supplied as a Bearer token (OpenAI-style), or
|
||||
// via the X-Api-Key or Api-Key headers (Anthropic/Azure OpenAI-style), so
|
||||
// clients built against any of those conventions work unmodified.
|
||||
func authMiddleware(apiKeys []string) bunrouter.MiddlewareFunc {
|
||||
if len(apiKeys) == 0 {
|
||||
return func(next bunrouter.HandlerFunc) bunrouter.HandlerFunc { return next }
|
||||
@@ -96,16 +165,27 @@ func authMiddleware(apiKeys []string) bunrouter.MiddlewareFunc {
|
||||
}
|
||||
return func(next bunrouter.HandlerFunc) bunrouter.HandlerFunc {
|
||||
return func(w http.ResponseWriter, req bunrouter.Request) error {
|
||||
token := strings.TrimPrefix(req.Header.Get("Authorization"), "Bearer ")
|
||||
if _, ok := keySet[token]; !ok {
|
||||
http.Error(w, "unauthorized", http.StatusUnauthorized)
|
||||
return nil
|
||||
token := requestAPIKey(req.Request)
|
||||
if _, ok := keySet[token]; !ok || token == "" {
|
||||
return writeError(w, http.StatusUnauthorized, "unauthorized")
|
||||
}
|
||||
return next(w, req)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// requestAPIKey extracts the client-supplied API key, checking (in order)
|
||||
// the Authorization Bearer scheme, X-Api-Key, and Api-Key headers.
|
||||
func requestAPIKey(req *http.Request) string {
|
||||
if auth := req.Header.Get("Authorization"); auth != "" {
|
||||
return strings.TrimPrefix(auth, "Bearer ")
|
||||
}
|
||||
if key := req.Header.Get("X-Api-Key"); key != "" {
|
||||
return key
|
||||
}
|
||||
return req.Header.Get("Api-Key")
|
||||
}
|
||||
|
||||
// traceMiddleware injects a *RequestTrace into every request context.
|
||||
func traceMiddleware() bunrouter.MiddlewareFunc {
|
||||
return func(next bunrouter.HandlerFunc) bunrouter.HandlerFunc {
|
||||
@@ -161,25 +241,22 @@ func loggingMiddleware(logger *zap.Logger) bunrouter.MiddlewareFunc {
|
||||
}
|
||||
}
|
||||
|
||||
// metricsKeyMiddleware guards a bunrouter.HandlerFunc with a dedicated Bearer token.
|
||||
// metricsKeyMiddleware guards a bunrouter.HandlerFunc with a dedicated API key
|
||||
// (Bearer, X-Api-Key, or Api-Key — see requestAPIKey).
|
||||
func metricsKeyMiddleware(apiKey string, h bunrouter.HandlerFunc) bunrouter.HandlerFunc {
|
||||
return func(w http.ResponseWriter, req bunrouter.Request) error {
|
||||
token := strings.TrimPrefix(req.Header.Get("Authorization"), "Bearer ")
|
||||
if token != apiKey {
|
||||
http.Error(w, "unauthorized", http.StatusUnauthorized)
|
||||
return nil
|
||||
if requestAPIKey(req.Request) != apiKey {
|
||||
return writeError(w, http.StatusUnauthorized, "unauthorized")
|
||||
}
|
||||
return h(w, req)
|
||||
}
|
||||
}
|
||||
|
||||
// metricsAuthHandler wraps a standard http.Handler with Bearer token auth.
|
||||
// metricsAuthHandler wraps a standard http.Handler with the same API key auth.
|
||||
func metricsAuthHandler(apiKey string, h http.Handler) bunrouter.HandlerFunc {
|
||||
return func(w http.ResponseWriter, req bunrouter.Request) error {
|
||||
token := strings.TrimPrefix(req.Header.Get("Authorization"), "Bearer ")
|
||||
if token != apiKey {
|
||||
http.Error(w, "unauthorized", http.StatusUnauthorized)
|
||||
return nil
|
||||
if requestAPIKey(req.Request) != apiKey {
|
||||
return writeError(w, http.StatusUnauthorized, "unauthorized")
|
||||
}
|
||||
h.ServeHTTP(w, req.Request)
|
||||
return nil
|
||||
@@ -196,3 +273,9 @@ func (sw *statusWriter) WriteHeader(code int) {
|
||||
sw.status = code
|
||||
sw.ResponseWriter.WriteHeader(code)
|
||||
}
|
||||
|
||||
func (sw *statusWriter) Flush() {
|
||||
if f, ok := sw.ResponseWriter.(http.Flusher); ok {
|
||||
f.Flush()
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,7 +1,11 @@
|
||||
openapi: "3.1.0"
|
||||
info:
|
||||
title: vecna Embedding Adapter
|
||||
description: Proxies text to a backing embedding model and adapts the result vectors between dimensions.
|
||||
description: >
|
||||
Proxies text to a backing embedding model and adapts the result vectors
|
||||
between dimensions. Also forwards the rest of the OpenAI-compatible API
|
||||
surface (chat, completions, models, audio, images, moderations, rerank)
|
||||
verbatim to the resolved target, with no adaptation applied.
|
||||
version: "1.0.0"
|
||||
|
||||
servers:
|
||||
@@ -21,7 +25,14 @@ components:
|
||||
type: object
|
||||
properties:
|
||||
error:
|
||||
type: object
|
||||
properties:
|
||||
message:
|
||||
type: string
|
||||
type:
|
||||
type: string
|
||||
code:
|
||||
type: integer
|
||||
|
||||
OpenAIEmbedRequest:
|
||||
type: object
|
||||
@@ -169,6 +180,40 @@ paths:
|
||||
"502":
|
||||
description: Backing model error
|
||||
|
||||
get:
|
||||
summary: OpenAI-compatible embeddings (query-param convenience form)
|
||||
operationId: openaiEmbeddingsGet
|
||||
parameters:
|
||||
- name: input
|
||||
in: query
|
||||
required: true
|
||||
description: Repeatable — one or more texts to embed.
|
||||
schema:
|
||||
type: array
|
||||
items:
|
||||
type: string
|
||||
- name: model
|
||||
in: query
|
||||
schema:
|
||||
type: string
|
||||
responses:
|
||||
"200":
|
||||
description: Adapted embeddings
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
$ref: '#/components/schemas/OpenAIEmbedResponse'
|
||||
"400":
|
||||
description: Bad request
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
$ref: '#/components/schemas/Error'
|
||||
"401":
|
||||
description: Unauthorized
|
||||
"502":
|
||||
description: Backing model error
|
||||
|
||||
/v1/models/{model}:embedContent:
|
||||
post:
|
||||
summary: Google-compatible single embedContent
|
||||
@@ -250,3 +295,158 @@ paths:
|
||||
description: Unauthorized
|
||||
"502":
|
||||
description: Backing model error
|
||||
|
||||
# --- Generic OpenAI-compatible passthrough -------------------------------
|
||||
# Forwarded verbatim to the resolved target (chosen by the body's "model"
|
||||
# field, falling back to forward.default). No vecna-specific processing —
|
||||
# request/response shape is whatever the backing model's API defines.
|
||||
# Each also exists under /map/{mapping}/... to force a specific target.
|
||||
/v1/models:
|
||||
get:
|
||||
summary: List models (passthrough)
|
||||
operationId: listModels
|
||||
responses:
|
||||
"200":
|
||||
description: Backend-defined model list
|
||||
"401":
|
||||
description: Unauthorized
|
||||
"502":
|
||||
description: Backing model error
|
||||
|
||||
/v1/models/{model}:
|
||||
get:
|
||||
summary: Retrieve a model (passthrough)
|
||||
operationId: retrieveModel
|
||||
parameters:
|
||||
- name: model
|
||||
in: path
|
||||
required: true
|
||||
schema:
|
||||
type: string
|
||||
responses:
|
||||
"200":
|
||||
description: Backend-defined model object
|
||||
"401":
|
||||
description: Unauthorized
|
||||
"502":
|
||||
description: Backing model error
|
||||
|
||||
/v1/chat/completions:
|
||||
post:
|
||||
summary: Chat completions (passthrough, streaming supported)
|
||||
operationId: chatCompletions
|
||||
responses:
|
||||
"200":
|
||||
description: Backend-defined chat completion (or SSE stream if "stream":true)
|
||||
"401":
|
||||
description: Unauthorized
|
||||
"502":
|
||||
description: Backing model error
|
||||
|
||||
/v1/completions:
|
||||
post:
|
||||
summary: Legacy completions (passthrough, streaming supported)
|
||||
operationId: completions
|
||||
responses:
|
||||
"200":
|
||||
description: Backend-defined completion (or SSE stream if "stream":true)
|
||||
"401":
|
||||
description: Unauthorized
|
||||
"502":
|
||||
description: Backing model error
|
||||
|
||||
/v1/moderations:
|
||||
post:
|
||||
summary: Moderations (passthrough)
|
||||
operationId: moderations
|
||||
responses:
|
||||
"200":
|
||||
description: Backend-defined moderation result
|
||||
"401":
|
||||
description: Unauthorized
|
||||
"502":
|
||||
description: Backing model error
|
||||
|
||||
/v1/images/generations:
|
||||
post:
|
||||
summary: Image generation (passthrough)
|
||||
operationId: imageGenerations
|
||||
responses:
|
||||
"200":
|
||||
description: Backend-defined image result
|
||||
"401":
|
||||
description: Unauthorized
|
||||
"502":
|
||||
description: Backing model error
|
||||
|
||||
/v1/images/edits:
|
||||
post:
|
||||
summary: Image edits (passthrough, multipart/form-data)
|
||||
operationId: imageEdits
|
||||
responses:
|
||||
"200":
|
||||
description: Backend-defined image result
|
||||
"401":
|
||||
description: Unauthorized
|
||||
"502":
|
||||
description: Backing model error
|
||||
|
||||
/v1/images/variations:
|
||||
post:
|
||||
summary: Image variations (passthrough, multipart/form-data)
|
||||
operationId: imageVariations
|
||||
responses:
|
||||
"200":
|
||||
description: Backend-defined image result
|
||||
"401":
|
||||
description: Unauthorized
|
||||
"502":
|
||||
description: Backing model error
|
||||
|
||||
/v1/audio/speech:
|
||||
post:
|
||||
summary: Text-to-speech (passthrough)
|
||||
operationId: audioSpeech
|
||||
responses:
|
||||
"200":
|
||||
description: Backend-defined audio bytes
|
||||
"401":
|
||||
description: Unauthorized
|
||||
"502":
|
||||
description: Backing model error
|
||||
|
||||
/v1/audio/transcriptions:
|
||||
post:
|
||||
summary: Speech-to-text (passthrough, multipart/form-data)
|
||||
operationId: audioTranscriptions
|
||||
responses:
|
||||
"200":
|
||||
description: Backend-defined transcription
|
||||
"401":
|
||||
description: Unauthorized
|
||||
"502":
|
||||
description: Backing model error
|
||||
|
||||
/v1/audio/translations:
|
||||
post:
|
||||
summary: Speech translation (passthrough, multipart/form-data)
|
||||
operationId: audioTranslations
|
||||
responses:
|
||||
"200":
|
||||
description: Backend-defined translation
|
||||
"401":
|
||||
description: Unauthorized
|
||||
"502":
|
||||
description: Backing model error
|
||||
|
||||
/v1/rerank:
|
||||
post:
|
||||
summary: Rerank (passthrough — Cohere/Infinity/vLLM-style)
|
||||
operationId: rerank
|
||||
responses:
|
||||
"200":
|
||||
description: Backend-defined rerank result
|
||||
"401":
|
||||
description: Unauthorized
|
||||
"502":
|
||||
description: Backing model error
|
||||
|
||||
Reference in New Issue
Block a user