wdevs/vecna
5
0
Fork 0
mirror of https://github.com/Warky-Devs/vecna.git synced 2026-05-05 01:26:58 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(adapter): add passthrough adapter for unchanged vectors

Browse Source
This commit is contained in:
Warky
2026-04-11 20:57:44 +02:00
parent 5c070e441e
commit c5608bf5cd
4 changed files with 57 additions and 32 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
cmd/vecna/convert.go
View File

@@ -95,6 +95,9 @@ func openWriter(path string) (io.Writer, error) {
// buildAdapter constructs the Adapter from the loaded config. // buildAdapter constructs the Adapter from the loaded config.
func buildAdapter(cfg *config.Config) (adapter.Adapter, error) { func buildAdapter(cfg *config.Config) (adapter.Adapter, error) {
ac := cfg.Adapter ac := cfg.Adapter
if ac.SourceDim == 0 && ac.TargetDim == 0 {
return adapter.NewPassthrough(), nil
}
switch ac.Type { switch ac.Type {
case "truncate": case "truncate":
tm, pm, err := parseTruncateModes(ac.TruncateMode, ac.PadMode) tm, pm, err := parseTruncateModes(ac.TruncateMode, ac.PadMode)

2
go.mod
View File

@@ -3,6 +3,7 @@ module github.com/Warky-Devs/vecna.git
go 1.26.1 go 1.26.1
require ( require (
github.com/go-viper/mapstructure/v2 v2.4.0
github.com/prometheus/client_golang v1.23.2 github.com/prometheus/client_golang v1.23.2
github.com/spf13/cobra v1.10.2 github.com/spf13/cobra v1.10.2
github.com/spf13/viper v1.21.0 github.com/spf13/viper v1.21.0
@@ -16,7 +17,6 @@ require (
github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect github.com/davecgh/go-spew v1.1.1 // indirect
github.com/fsnotify/fsnotify v1.9.0 // indirect github.com/fsnotify/fsnotify v1.9.0 // indirect
github.com/go-viper/mapstructure/v2 v2.4.0 // indirect
github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
github.com/pelletier/go-toml/v2 v2.2.4 // indirect github.com/pelletier/go-toml/v2 v2.2.4 // indirect

11
pkg/adapter/adapter.go
View File

@@ -41,6 +41,17 @@ type Adapter interface {
TargetDim() int TargetDim() int
} }
// passthroughAdapter returns the input vector unchanged.
type passthroughAdapter struct{}
func (passthroughAdapter) Adapt(vec []float32) ([]float32, error) { return vec, nil }
func (passthroughAdapter) SourceDim() int { return 0 }
func (passthroughAdapter) TargetDim() int { return 0 }
// NewPassthrough returns an Adapter that returns vectors unchanged.
// Use when no dimension adaptation is needed.
func NewPassthrough() Adapter { return passthroughAdapter{} }
// NewTruncate returns a TruncateAdapter for Matryoshka-style or simple truncation/padding. // NewTruncate returns a TruncateAdapter for Matryoshka-style or simple truncation/padding.
// t controls which end is dropped when downscaling; p controls which end is padded when upscaling. // t controls which end is dropped when downscaling; p controls which end is padded when upscaling.
func NewTruncate(sourceDim, targetDim int, t TruncateMode, p PadMode) (Adapter, error) { func NewTruncate(sourceDim, targetDim int, t TruncateMode, p PadMode) (Adapter, error) {

73
pkg/config/config.go
View File

@@ -6,65 +6,66 @@ import (
"os" "os"
"strings" "strings"
"github.com/go-viper/mapstructure/v2"
"github.com/spf13/viper" "github.com/spf13/viper"
) )
// Config is the root configuration for vecna. // Config is the root configuration for vecna.
type Config struct { type Config struct {
Server ServerConfig `mapstructure:"server"` Server ServerConfig `mapstructure:"server" json:"server" yaml:"server" xml:"server"`
Metrics MetricsConfig `mapstructure:"metrics"` Metrics MetricsConfig `mapstructure:"metrics" json:"metrics" yaml:"metrics" xml:"metrics"`
Forward ForwardConfig `mapstructure:"forward"` Forward ForwardConfig `mapstructure:"forward" json:"forward" yaml:"forward" xml:"forward"`
Adapter AdapterConfig `mapstructure:"adapter"` Adapter AdapterConfig `mapstructure:"adapter" json:"adapter" yaml:"adapter" xml:"adapter"`
} }
// ServerConfig controls the HTTP listener and inbound auth. // ServerConfig controls the HTTP listener and inbound auth.
type ServerConfig struct { type ServerConfig struct {
Port int `mapstructure:"port"` Port int `mapstructure:"port" json:"port" yaml:"port" xml:"port"`
Host string `mapstructure:"host"` Host string `mapstructure:"host" json:"host" yaml:"host" xml:"host"`
APIKeys []string `mapstructure:"api_keys"` APIKeys []string `mapstructure:"api_keys" json:"api_keys" yaml:"api_keys" xml:"api_keys"`
} }
// MetricsConfig controls Prometheus metrics exposure. // MetricsConfig controls Prometheus metrics exposure.
type MetricsConfig struct { type MetricsConfig struct {
Enabled bool `mapstructure:"enabled"` Enabled bool `mapstructure:"enabled" json:"enabled" yaml:"enabled" xml:"enabled"`
Path string `mapstructure:"path"` Path string `mapstructure:"path" json:"path" yaml:"path" xml:"path"`
APIKey string `mapstructure:"api_key"` APIKey string `mapstructure:"api_key" json:"api_key" yaml:"api_key" xml:"api_key"`
} }
// ForwardConfig holds all named forwarding targets. // ForwardConfig holds all named forwarding targets.
type ForwardConfig struct { type ForwardConfig struct {
Default string `mapstructure:"default"` Default string `mapstructure:"default" json:"default" yaml:"default" xml:"default"`
Targets map[string]ForwardTarget `mapstructure:"targets"` Targets map[string]ForwardTarget `mapstructure:"targets" json:"targets" yaml:"targets" xml:"targets"`
} }
// ForwardTarget is a named backing embedding model with one or more endpoints. // ForwardTarget is a named backing embedding model with one or more endpoints.
type ForwardTarget struct { type ForwardTarget struct {
Endpoints []EndpointConfig `mapstructure:"endpoints"` Endpoints []EndpointConfig `mapstructure:"endpoints" json:"endpoints" yaml:"endpoints" xml:"endpoints"`
Model string `mapstructure:"model"` Model string `mapstructure:"model" json:"model" yaml:"model" xml:"model"`
APIKey string `mapstructure:"api_key"` APIKey string `mapstructure:"api_key" json:"api_key" yaml:"api_key" xml:"api_key"`
APIType string `mapstructure:"api_type"` APIType string `mapstructure:"api_type" json:"api_type" yaml:"api_type" xml:"api_type"`
TimeoutSecs int `mapstructure:"timeout_secs"` TimeoutSecs int `mapstructure:"timeout_secs" json:"timeout_secs" yaml:"timeout_secs" xml:"timeout_secs"`
CooldownSecs int `mapstructure:"cooldown_secs"` CooldownSecs int `mapstructure:"cooldown_secs" json:"cooldown_secs" yaml:"cooldown_secs" xml:"cooldown_secs"`
PriorityDecay int `mapstructure:"priority_decay"` PriorityDecay int `mapstructure:"priority_decay" json:"priority_decay" yaml:"priority_decay" xml:"priority_decay"`
PriorityRecovery int `mapstructure:"priority_recovery"` PriorityRecovery int `mapstructure:"priority_recovery" json:"priority_recovery" yaml:"priority_recovery" xml:"priority_recovery"`
} }
// EndpointConfig is a single URL within a ForwardTarget. // EndpointConfig is a single URL within a ForwardTarget.
type EndpointConfig struct { type EndpointConfig struct {
URL string `mapstructure:"url"` URL string `mapstructure:"url" json:"url" yaml:"url" xml:"url"`
Priority int `mapstructure:"priority"` Priority int `mapstructure:"priority" json:"priority" yaml:"priority" xml:"priority"`
APIKey string `mapstructure:"api_key"` APIKey string `mapstructure:"api_key" json:"api_key" yaml:"api_key" xml:"api_key"`
} }
// AdapterConfig selects and tunes the dimension adapter. // AdapterConfig selects and tunes the dimension adapter.
type AdapterConfig struct { type AdapterConfig struct {
Type string `mapstructure:"type"` Type string `mapstructure:"type" json:"type" yaml:"type" xml:"type"`
SourceDim int `mapstructure:"source_dim"` SourceDim int `mapstructure:"source_dim" json:"source_dim" yaml:"source_dim" xml:"source_dim"`
TargetDim int `mapstructure:"target_dim"` TargetDim int `mapstructure:"target_dim" json:"target_dim" yaml:"target_dim" xml:"target_dim"`
TruncateMode string `mapstructure:"truncate_mode"` TruncateMode string `mapstructure:"truncate_mode" json:"truncate_mode" yaml:"truncate_mode" xml:"truncate_mode"`
PadMode string `mapstructure:"pad_mode"` PadMode string `mapstructure:"pad_mode" json:"pad_mode" yaml:"pad_mode" xml:"pad_mode"`
Seed int64 `mapstructure:"seed"` Seed int64 `mapstructure:"seed" json:"seed" yaml:"seed" xml:"seed"`
MatrixFile string `mapstructure:"matrix_file"` MatrixFile string `mapstructure:"matrix_file" json:"matrix_file" yaml:"matrix_file" xml:"matrix_file"`
} }
// extensions viper will detect automatically. // extensions viper will detect automatically.
@@ -132,7 +133,17 @@ func Load(cfgFile string) (*Config, error) {
} }
var cfg Config var cfg Config
if err := v.Unmarshal(&cfg); err != nil { if err := v.Unmarshal(&cfg, func(dc *mapstructure.DecoderConfig) {
// Viper lowercases all keys (e.g. "SourceDim" → "sourcedim"), but
// mapstructure tags use snake_case ("source_dim"). Strip underscores
// before comparing so both forms resolve correctly.
dc.MatchName = func(mapKey, fieldName string) bool {
norm := func(s string) string {
return strings.ToLower(strings.ReplaceAll(s, "_", ""))
}
return norm(mapKey) == norm(fieldName)
}
}); err != nil {
return nil, fmt.Errorf("unmarshal config: %w", err) return nil, fmt.Errorf("unmarshal config: %w", err)
} }