Compare commits

..
200 Commits
Author SHA1 Message Date
Hein 3bd3e46409 fix(restheadspec): handle primary key changes in updates
* Allow primary key changes when specified in the request body.
* Ensure correct record fetching after primary key updates.
* Add integration tests for primary key update scenarios.
2026-10-01 10:16:26 +02:00
Hein 247111c32e docs(README): update table of contents and feature descriptions 2026-10-01 09:46:17 +02:00
Hein Puth (Warkanum) ec8d4d2c77 Merge pull request #25 from bitechdev/fix/row-security-update-delete
Fix/row security update delete
2026-10-01 09:40:33 +02:00
Hein a3287f3b53 fix(security): reorder import statements for consistency 2026-10-01 09:33:57 +02:00
Hein 1e4a76643d fix(security): apply row security to update and delete queries
ApplyRowSecurity only accepted common.SelectQuery, so the BeforeScan hook
failed closed on update/delete when a row-security template existed.
Type-switch on SelectQuery, UpdateQuery and DeleteQuery; other types
still return an error.
2026-10-01 09:31:56 +02:00
Hein Puth (Warkanum) a81031b83d Merge pull request #23 from bitechdev/fix/db-connection-bursts
Tests / Race Detector (push) Failing after 23s
Tests / Unit Tests (push) Failing after 25s
Tests / Integration Tests (push) Failing after 26s
Build , Vet Test, and Lint / Build (push) Successful in 1m4s
Build , Vet Test, and Lint / Run Vet Tests (1.23.x) (push) Successful in 1m34s
Build , Vet Test, and Lint / Run Vet Tests (1.24.x) (push) Successful in 1m34s
Build , Vet Test, and Lint / Lint Code (push) Failing after 1m34s
Fix/db connection bursts
2026-09-30 23:53:17 +02:00
warkanum ac4cf9b4b6 Merge branch 'main' of github.com:bitechdev/ResolveSpec into fix/db-connection-bursts 2026-09-30 23:51:24 +02:00
warkanum b35399fdfa feat(security): exclude hidden/masked columns from create and update payloads 2026-09-30 23:48:09 +02:00
warkanum a65ca5f5ce fix: fire resolvespec AfterRead/AfterCreate/AfterDelete, key restheadspec total cache by record id 2026-09-30 23:40:32 +02:00
warkanum 5933637a88 docs(readme): update slogan placement in README 2026-09-30 23:35:08 +02:00
warkanum 7f84debdc5 test(tx): regression tests for per-request transactions across all specs 2026-09-30 23:19:58 +02:00
warkanum 2042205817 fix(pgsql): return subquery preload errors instead of logging and continuing 2026-09-30 23:11:30 +02:00
warkanum 6335bfe87e docs(readme): reference all pkg packages and clients 2026-09-30 23:04:56 +02:00
warkanum 129c1a043d docs(readme): document single transaction per request, OnTxBegin, test server 2026-09-30 23:03:50 +02:00
warkanum da0b1f5123 chore(testserver): host networking, ports 8123/8124, smoke read+update, dbtrace pooled=0 verified 2026-09-30 23:01:52 +02:00
warkanum ff76eb8e1f feat(security): stamp transaction-local settings on OnTxBegin in all specs 2026-09-30 22:55:26 +02:00
warkanum 3b93802a25 fix(tx): run restheadspec AfterRead in a second short transaction 2026-09-30 22:53:07 +02:00
warkanum 6b6f540ab0 feat(tx): run funcspec OnTxBegin and BeforeResponse in transactions 2026-09-30 22:50:41 +02:00
warkanum 4cbe4f597d feat(tx): run resolvemcp operations in per-operation transactions with OnTxBegin 2026-09-30 22:49:47 +02:00
warkanum ed457eb14a feat(tx): run websocketspec and mqttspec operations in per-message transactions
Reads and deletes run in one transaction; create and update write in one
and re-fetch plus After hooks in a second. OnTxBegin fires first in each.
2026-09-30 22:46:46 +02:00
warkanum 97bcb44fdc fix(clients): verify C# client, read Content-Range from content headers 2026-09-30 22:44:15 +02:00
warkanum 17bb6ea76d fix(clients): verify Dart client, case-insensitive Content-Range lookup 2026-09-30 22:42:55 +02:00
warkanum ce706bacda feat(tx): run update re-fetch and post-commit hooks in a second transaction
Re-fetch, BeforeScan and AfterUpdate/AfterCreate now run on a short
transaction that fires OnTxBegin. Existence selects inside the first
transaction use tx instead of the pool.
2026-09-30 22:42:33 +02:00
warkanum eb492d52aa feat(clients): add Go, Rust, C# and Dart clients for ResolveSpec and FunctionSpec 2026-09-30 22:40:33 +02:00
warkanum f2dbe2561c feat(hooks): add OnTxBegin and runInTx for resolvespec and restheadspec
Every transaction the handlers open now fires OnTxBegin first, with the
transaction in hookCtx.Tx, via common.RunRequestTx.
2026-09-30 22:40:06 +02:00
warkanum cd96404cdd fix(delete): run delete hooks and queries in one transaction
- resolvespec/restheadspec: single and batch delete use one transaction
- add sqlmock tests for delete transaction behaviour
- testmodels: serial integer ids; update tests accordingly
- add compose testserver, smoke script, podman-first Makefile targets
2026-09-30 22:33:25 +02:00
warkanum b2b815552f refactor: move JS and Python clients under clients/ 2026-09-30 22:31:43 +02:00
warkanum f6a9daa89e docs(audit): add plan for single transaction per request 2026-09-30 22:18:24 +02:00
warkanum 54e6a3b17c feat(resolvespec-python): add Python client for ResolveSpec, HeaderSpec, FunctionSpec and WebSocketSpec 2026-09-30 22:18:01 +02:00
warkanum ab3d2b5b04 docs(audit): add funcspec server-side audit 2026-09-30 22:18:01 +02:00
Hein Puth (Warkanum) 9c4d916490 Merge pull request #22 from bitechdev/fix/db-connection-bursts
Tests / Unit Tests (push) Failing after 28s
Tests / Race Detector (push) Failing after 29s
Tests / Integration Tests (push) Failing after 28s
Build , Vet Test, and Lint / Build (push) Successful in 1m7s
Build , Vet Test, and Lint / Run Vet Tests (1.24.x) (push) Successful in 1m38s
Build , Vet Test, and Lint / Lint Code (push) Failing after 1m39s
Build , Vet Test, and Lint / Run Vet Tests (1.23.x) (push) Successful in 1m39s
Fix/db connection bursts
2026-09-30 21:45:20 +02:00
warkanum 3e327d0c78 fix(db): reduce per-request connection bursts and add dbtrace
* Throttle async session-activity writes to once per token per minute
* Add singleflight to session lookups, keystore validation and
  column/row security loads to stop cold-cache stampedes
* Preload security rules in BeforeHandle (restheadspec, resolvespec) so
  they no longer need a second connection while the read tx is open
* Add pkg/dbtrace: opt-in per-request DB call counting and pool logging
  (db_trace.* config, RESOLVESPEC_DB_TRACE_* env), wired into testserver
* Add tests for load dedup, activity throttle and dbtrace
2026-09-30 21:44:28 +02:00
warkanum 62cc14c02a feat(resolvespec-js): support extended restheadspec headers
Add HeaderSpecOptions, vector_search, X-Preload-Where, X-Expand,
custom SQL joins/or, spatial/vector filters, response format, flags
and X-Files to buildHeaders, types and README.
2026-09-30 21:37:41 +02:00
Hein 4c5dffc3d1 test(mqttspec): add tests for update behavior with empty strings 2026-09-30 17:15:55 +02:00
Hein 2898b335f8 feat(db): add ApplicationName to connection configuration
* Introduced ApplicationName field to identify clients in DSN
* Set default ApplicationName to "ResolveSpec"
* Updated tests for ApplicationName handling in DSN
2026-09-30 17:12:04 +02:00
Hein 20ba8ed112 feat(update): allow clearing values with "" and null on update
Update handlers skipped empty strings and nulls, so a client could not
blank or null out a column. Every key present in the payload now
overwrites the stored value, including "" and null.

- add common.MergeUpdateValues and use it in resolvespec and restheadspec
- add Handler.SetDisallowNulls to skip null values (""still overwrites)
- websocketspec and mqttspec now write only the keys present in the
  payload via SetMap instead of updating the whole zeroed model, which
  clobbered absent fields
2026-09-30 17:03:42 +02:00
Hein 1214f69e0c feat(middleware): export clientqueue_enqueued_total counter
Counts every request placed in a wait queue regardless of outcome, so the
total ever queued no longer has to be summed from queued, timeout and
canceled.
2026-09-30 16:16:25 +02:00
Hein 89a58ab3a0 feat(middleware): export clientqueue_waiting_clients gauge
Counts clients with at least one queued request, updated whenever a
client's waiter count crosses zero (enqueue, grant, timeout, cancel).
2026-09-30 16:14:27 +02:00
Hein 48081b4aa4 docs: document client queue and dbmanager pool stats 2026-09-30 15:46:43 +02:00
Hein 467dbd66c8 feat(middleware): add per-client FIFO request queue with burst metrics
ClientQueue limits concurrent requests per client (X-Client-Id, then
Authorization, session, IP) and queues the rest first-in-first-out, with
bounded depth, max wait and idle eviction. Chain composes it with auth for
the Setup*Routes middleware slot. Exports burst, wait and depth metrics.
The test server uses it with a limit of 10.
2026-09-30 15:46:42 +02:00
Hein 178d40587d feat(dbmanager): report pool max size and total connections opened
Add MaxOpenConnections to provider and connection stats, exposed as the
max state of dbmanager_connection_pool_size. Count physical dials through a
counting driver connector and export dbmanager_connections_opened_total;
existing_db pools derive an approximate count from sql.DBStats.
2026-09-30 15:46:41 +02:00
Hein d3a99550d9 fix(security): name GetTemplate results to satisfy gocritic
Tests / Race Detector (push) Failing after 25s
Tests / Unit Tests (push) Failing after 25s
Tests / Integration Tests (push) Failing after 27s
Build , Vet Test, and Lint / Build (push) Successful in 1m3s
Build , Vet Test, and Lint / Run Vet Tests (1.24.x) (push) Successful in 1m30s
Build , Vet Test, and Lint / Lint Code (push) Successful in 1m31s
Build , Vet Test, and Lint / Run Vet Tests (1.23.x) (push) Successful in 1m31s
2026-09-30 13:46:03 +02:00
Hein 8a94d884e7 fix(security): verify passwords, bind row-security args, fail closed on panic
Verify bcrypt passwords in Direct mode and the shipped procedures, hash on
register/reset, ignore client-supplied roles and level at registration and
drop the password from the jwt_login payload. Legacy cleartext upgrade is
opt-in. Row security templates now bind the user as a parameter, validate
identifiers, attach via common.SelectQuery and fail the request if the
filter cannot be attached. ApplyColumnSecurity and GetRowSecurityTemplate
convert panics to errors and the hooks fail closed. Update audit status.
2026-09-30 13:44:59 +02:00
Hein f9c948ca4e fix(tracing): address audit findings
Default to TLS export with Insecure/TLSConfig/Headers options, parent-based
ratio sampling (default 0.1), and no query string or Host in span attributes.
Name spans by route template, record status and panics (re-raised), guard the
tracer with atomic.Pointer, reject double init, add init timeout and attribute
length limit, and move to semconv v1.26.0. Add tracing.insecure and
tracing.sample_rate config keys and tests.
2026-09-30 13:39:50 +02:00
Hein 164ba2b240 fix(security): remove lock contention, cache rules, stop leaked goroutines
Load column/row security without holding locks across provider calls,
honour pOverwrite with a 30s TTL and pruning, cap tokens per
Authorization header, detach session activity updates with a timeout,
stop OAuth2 cleanup goroutines via Close, move nil-map checks inside
locks, and drop O(n^2) string building. Update audit status.
2026-09-30 13:31:03 +02:00
Hein 97fe88b3a6 fix(modelregistry): address audit findings
Replace try-lock/sleep scheme with blocking locks, add ErrModelNotFound/
ErrModelExists/ErrInvalidModel sentinels, make RegisterModelWithRules
atomic, snapshot in IterateModels, guard defaultRegistry access, cap the
pointer-unwrap depth, and recover panics in callbacks and reflection.
Security hooks now allow-by-default only on ErrModelNotFound. Add tests.
2026-09-30 13:26:55 +02:00
Hein a4e1abc1df fix(logger): address audit findings
Redact and rate-limit error tracker fan-out, cap panic stack capture,
sanitise stdlib fallback output, strip contexts in Info/Debug, sync the
replaced logger, add Sync, UpdateLoggerE and CatchPanicRethrow, cache
the PID, and add tests.
2026-09-30 13:19:52 +02:00
Hein d7cb111496 docs(audit): add audit report for pkg/common 2026-09-30 13:16:47 +02:00
Hein f66930c3c9 chore(gosec): enable gosec and address findings 2026-09-30 13:16:12 +02:00
Hein 9533c3a0ed fix(cache): harden providers and default cache handling
Make cache-write failures non-fatal in GetOrSet/Remember, replace the
unsynchronised default cache with an atomic pointer, fix tag-index leaks
and write-lock-on-read in the memory provider, add a janitor, closed
state and byte copies, hash/namespace memcache keys with CAS tag index,
gate Clear behind AllowFlush, return ErrNotFound without the key, and
allowlist Redis stats. Mark audit status.
2026-09-30 13:13:06 +02:00
Hein 652621a70e ci: add race detector job 2026-09-30 13:09:44 +02:00
Hein c7b4530689 fix(race): make make test-race pass across ./pkg/...
Add a test-race target and run test-unit over ./pkg/...

Production races:
- logger: guard Logger/errorTracker with an RWMutex
- security: copy UserContext for the async session-activity goroutine
  and track it with a WaitGroup

Test fixes:
- eventbroker, websocketspec: use atomics for state shared with workers
- security: wait for async activity updates before touching sqlmock
- mqttspec: build full HookContext, set SubscriptionID, pin the
  in-memory SQLite to one connection

Update the cross-cutting audit (X1) with status and findings.
2026-09-30 13:07:47 +02:00
Hein e1cf72834e fix(config): lock Manager, harden defaults handling and path/IP helpers
Guard viper with an RWMutex, make the singleton race-free and stop
NewManager replacing the global (add SetConfigManager), write saved
configs 0600, search CWD last, add ConfigFileUsed and Config.Validate,
nil-safe PathsConfig.Set, confine PathsConfig.Join, bound GetIPs DNS
lookup, and drop dead Unmarshal in SetConfig. Mark audit status.
2026-09-30 13:02:40 +02:00
HeinandClaude Sonnet 5.5 3657aa94cc fix(dbmanager): explicitly ignore best-effort listener close errors
Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
2026-09-30 12:40:22 +02:00
HeinandClaude Sonnet 5.5 da1af1487e fix(dbmanager): keep the pool alive across errors and restarts
Implements the fixes from audit/pkg/dbmanager.audit.md.

- Stop closing the shared *sql.DB to recover from errors. Adapter
  factories and the health checker no longer call Reconnect; Reconnect is
  atomic and operator-only.
- Postgres uses a custom driver.Connector: Reconnect retires pooled
  connections by generation without closing the pool, so held Bun/GORM
  handles keep working. Verified against a live server restart.
- Add TCP keepalive, TCP_USER_TIMEOUT, a bounded reuse ping and
  statement_timeout as a runtime parameter; drop the 2 min timeout floor.
- Health check pings without holding the connection lock.
- Listener: single goroutine pair, bounded Close without UNLISTEN, and
  serialised use of the pgx connection (fixes conn busy and a close race).
- Fix Connect/Close/Connect/Close panic, idempotent Connect, dial outside
  the manager lock, clean up on partial failure.
- SQLite: pin :memory: to one connection, pragmas via DSN.
- Escape credentials in Postgres/MSSQL/Mongo DSNs; sslmode defaults to
  prefer. Wire retry settings, publish metrics, fix logger calls.
- NewConnectionFromDB: Close is a no-op with a warning (caller owns the
  pool); Reconnect only pings.
- Document correct usage in the README; mark the audit with what was done.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
2026-09-30 12:40:14 +02:00
Hein bc8bff7955 docs(audit): add audit reports for pkg/testmodels and pkg/tracing
Tests / Unit Tests (push) Failing after 24s
Tests / Integration Tests (push) Failing after 26s
Build , Vet Test, and Lint / Build (push) Successful in 1m7s
Build , Vet Test, and Lint / Run Vet Tests (1.23.x) (push) Successful in 1m31s
Build , Vet Test, and Lint / Lint Code (push) Successful in 1m33s
Build , Vet Test, and Lint / Run Vet Tests (1.24.x) (push) Successful in 1m34s
2026-09-29 17:15:00 +02:00
Hein a74eebc7f3 fix(json-columns): skip JSON select columns with no model scan target
Tests / Unit Tests (push) Failing after 28s
Tests / Integration Tests (push) Failing after 29s
Build , Vet Test, and Lint / Build (push) Successful in 1m12s
Build , Vet Test, and Lint / Run Vet Tests (1.24.x) (push) Successful in 1m43s
Build , Vet Test, and Lint / Run Vet Tests (1.23.x) (push) Successful in 1m46s
Build , Vet Test, and Lint / Lint Code (push) Successful in 1m48s
Requesting a JSON sub-field (e.g. jsonvalue->'product'->>'cost') that has
no matching bun scanonly field on the model made the whole read fail with
"bun: ModelX does not have column Y", since bun scans SELECT results
straight into the typed model struct.

Add reflection.HasColumn to check whether the model can actually receive
a given column (including scanonly fields, walking embedded structs), and
gate the JSON select-column expression on it in ApplySelectColumns
(shared by websocketspec/mqttspec) and the resolvespec/restheadspec
handlers. When there's no scan target, drop just that column with a
warning instead of erroring the whole request.
2026-09-28 12:30:56 +02:00
Hein 6687a7a5cd fix(bun): spread variadic args correctly in ColumnExpr 2026-09-28 12:01:38 +02:00
warkanum e8fbbede7e chore: update version to 1.0.2 and changelog
Tests / Unit Tests (push) Failing after 25s
Tests / Integration Tests (push) Failing after 26s
Build , Vet Test, and Lint / Build (push) Successful in 1m9s
Build , Vet Test, and Lint / Lint Code (push) Successful in 1m29s
Build , Vet Test, and Lint / Run Vet Tests (1.24.x) (push) Successful in 1m32s
Build , Vet Test, and Lint / Run Vet Tests (1.23.x) (push) Successful in 1m33s
2026-09-23 20:58:18 +02:00
warkanum 7f8982fa35 feat(headerspec): add UTF-8 encoding/decoding support
* Implement round-trip encoding/decoding for UTF-8 values in header functions.
* Update encodeHeaderValue and decodeHeaderValue to use base64 utility functions.
* Add tests for UTF-8 header value handling.
* Update Vite config to externalize base64 utility.
2026-09-23 20:57:50 +02:00
warkanum b587cbd3c4 feat(headers): add support for custom HTTP headers in clients
* Introduced `headers` property in `ClientConfig` interface.
* Updated `HeaderSpecClient` and `ResolveSpecClient` to utilize custom headers.
* Implemented `mergeHeaders` function to handle case-insensitive header merging.
* Added tests for custom header functionality in clients.
2026-09-23 19:47:05 +02:00
Hein a220338eea feat(spectypes): add CIString, LCString, and UCString types with tests
Tests / Unit Tests (push) Failing after 28s
Tests / Integration Tests (push) Failing after 30s
Build , Vet Test, and Lint / Build (push) Successful in 1m8s
Build , Vet Test, and Lint / Lint Code (push) Successful in 1m18s
Build , Vet Test, and Lint / Run Vet Tests (1.23.x) (push) Successful in 1m35s
Build , Vet Test, and Lint / Run Vet Tests (1.24.x) (push) Successful in 1m36s
2026-09-21 14:06:33 +02:00
Hein 20c67166d0 fix(handler): support implicit updates from request body 2026-09-21 11:18:10 +02:00
Hein 749dad4ed1 fix(quickproxy): ensure request body is preserved on fallback
Tests / Unit Tests (push) Failing after 26s
Tests / Integration Tests (push) Failing after 41s
Build , Vet Test, and Lint / Build (push) Successful in 4m26s
Build , Vet Test, and Lint / Lint Code (push) Successful in 4m58s
Build , Vet Test, and Lint / Run Vet Tests (1.24.x) (push) Successful in 5m1s
Build , Vet Test, and Lint / Run Vet Tests (1.23.x) (push) Successful in 5m3s
2026-09-21 09:21:58 +02:00
warkanum d6c5740f9c fix(handler): add operation type to hook context
Build , Vet Test, and Lint / Run Vet Tests (1.23.x) (push) Failing after 1s
Build , Vet Test, and Lint / Run Vet Tests (1.24.x) (push) Failing after 1s
Build , Vet Test, and Lint / Lint Code (push) Failing after 1s
Build , Vet Test, and Lint / Build (push) Failing after 1s
Tests / Unit Tests (push) Failing after 0s
Tests / Integration Tests (push) Failing after 10s
2026-09-20 16:12:44 +02:00
warkanum 817b781c88 fix(security): skip loading security rules if disabled 2026-09-20 15:52:25 +02:00
warkanum 87eaa9e18c fix(security): skip row security enforcement for specific operations
* Add ShouldSkipRowSecurity function to determine when to bypass row security
* Update ApplyRowSecurity to utilize operation context for enforcement
2026-09-20 15:51:02 +02:00
Hein 4f6878099b fix(quickproxy): reject Exclude entries outside their rule's URLPrefix
Tests / Unit Tests (push) Failing after 5s
Tests / Integration Tests (push) Failing after 23s
Build , Vet Test, and Lint / Build (push) Successful in 52s
Build , Vet Test, and Lint / Run Vet Tests (1.24.x) (push) Successful in 55s
Build , Vet Test, and Lint / Run Vet Tests (1.23.x) (push) Successful in 56s
Build , Vet Test, and Lint / Lint Code (push) Successful in 1m4s
An Exclude entry only ever matches requests that already fall under
its rule's URLPrefix, so one written without that prefix (e.g.
"/health" on a rule for "/api") silently never triggered. Validate
that each Exclude entry itself starts with the rule's URLPrefix,
failing NewService instead of accepting a no-op config.
2026-09-17 11:40:49 +02:00
Hein 0d8b136b91 feat(quickproxy): support per-rule Exclude path prefixes
Rule.Exclude lists path prefixes that should never be proxied by that
rule, even though they fall under its URLPrefix. A request matching an
Exclude prefix is treated as a non-match for that rule: matching
continues against other configured rules, falling back to the
caller-supplied handler if none apply. Lets a catch-all "/" rule proxy
everything except carved-out paths like "/health".
2026-09-17 11:38:02 +02:00
Hein 6de9be0ae7 chore(proxy): remove outdated proxy documentation 2026-09-17 11:09:14 +02:00
Hein 82e923b16e fix(quickproxy): use http.NotFoundHandler per golangci-lint gocritic 2026-09-17 11:08:12 +02:00
Hein 9a664593f0 feat(quickproxy): add reverse-proxy-with-static-fallback package
Adds pkg/server/quickproxy: longest-prefix rule matching over
net/http/httputil.ReverseProxy, falling back to a caller-supplied
handler when the upstream is unreachable or returns 404. Any other
upstream response streams through unchanged. All HTTP methods are
proxied, with a configurable global dial/response-header timeout
(quickproxy.WithTimeout, default 10s).

GoCore-side wiring (config field, webserver2/proxy.go, server.go
route ordering) is tracked separately in that repo.
2026-09-17 11:07:53 +02:00
Hein 6e3124e4e0 fix(restheadspec): prevent casting numeric values in ILIKE filters
Tests / Unit Tests (push) Failing after 5s
Tests / Integration Tests (push) Failing after 24s
Build , Vet Test, and Lint / Build (push) Successful in 54s
Build , Vet Test, and Lint / Run Vet Tests (1.23.x) (push) Successful in 1m6s
Build , Vet Test, and Lint / Lint Code (push) Successful in 2m38s
Build , Vet Test, and Lint / Run Vet Tests (1.24.x) (push) Successful in 2m47s
2026-09-15 13:55:11 +02:00
Hein d5de48011b fix(websocketspec,mqttspec,resolvemcp): stop casting citext columns to TEXT for LIKE/ILIKE
Same class of bug as the restheadspec/resolvespec fix: these handlers
unconditionally rendered CAST(col AS TEXT) LIKE/ILIKE for every column,
which flips a citext column to case-sensitive matching and defeats a
citext index. Thread the model through to buildFilterCondition/applyFilters
so reflection.IsCitextColumn can skip the cast for citext columns.

resolvemcp's eq/neq/gt/lt paths never cast (they never had the
restheadspec-style reflect.Kind cast heuristic), so this only touches
LIKE/ILIKE. funcspec is unaffected: it has no Go struct model to check
against (colname/value come straight from SQL function parameters).
2026-09-15 11:26:28 +02:00
Hein 6bd6a6f164 fix(restheadspec,resolvespec): stop casting SqlNull-wrapped and citext columns to TEXT in filters
reflect.Type.Kind() on spectypes.SqlNull[T] wrappers (SqlInt16/32/64,
SqlFloat64, SqlBool, SqlString, and embedders like SqlTimeStamp) always
reports reflect.Struct, never the wrapped T. ValidateAndAdjustFilterForColumnType
treated those as "complex" columns and forced CAST(col AS TEXT) on eq/gt/lt
filters, e.g. CAST(atdetail.rid_parent AS TEXT) = '90446096', which can't
use the index on rid_parent.

Add spectypes.UnwrapKind to see through SqlNull wrappers to the underlying
Kind, and use it in GetColumnTypeFromModel so numeric/string SqlNull columns
are recognized correctly and compared natively.

Also stop unconditionally casting to TEXT for LIKE/ILIKE and add
reflection.IsCitextColumn: citext columns are already case-insensitive, so
casting them to TEXT flips to case-sensitive matching and defeats a citext
index.
2026-09-15 11:18:13 +02:00
Hein 1885ce016b fix(spectypes): stop nulling out midnight SqlTime values; add sql_types tests
Tests / Unit Tests (push) Failing after 7s
Tests / Integration Tests (push) Failing after 23s
Build , Vet Test, and Lint / Lint Code (push) Failing after 5m46s
Build , Vet Test, and Lint / Build (push) Successful in 7m55s
Build , Vet Test, and Lint / Run Vet Tests (1.23.x) (push) Successful in 8m49s
Build , Vet Test, and Lint / Run Vet Tests (1.24.x) (push) Successful in 8m49s
SqlTime previously treated 00:00:00 as equivalent to null, which incorrectly
dropped legitimate midnight time values on marshal/unmarshal.

Also adds test coverage for SqlBool, generic SqlNull conversion helpers
(Int64/Float64/Bool/Time/UUID), FromString edge cases, NewSql, and the
SqlDate/SqlTimeStamp zero-value/sentinel handling.
2026-09-11 11:01:16 +02:00
Hein Puth (Warkanum) eeb7ba04d8 Merge pull request #21 from bitechdev/feat/json-column-support
Feat/json column support
2026-09-11 10:53:29 +02:00
Hein e957753ce4 chore(common): satisfy linter on json_column.go (named results, De Morgan) 2026-09-07 17:07:22 +02:00
Hein 206edd4bfd feat: add JSON/JSONB sub-field select, filter and sort
Support column references that traverse into JSON/JSONB values —
data->>'x', data#>>'{a,b}', data->'a'->>'b', and the dotted data.a.b
shorthand — in SELECT column lists, WHERE filters and ORDER BY, across
the restheadspec, resolvespec, websocketspec and mqttspec handlers.

- pkg/common/json_column.go: canonical ParseColumnRef + ColumnRef.SQL()
  builder. JSON path segments are bound as a single ?::text[] parameter,
  never interpolated; cast targets are whitelisted via NormalizeCastTarget.
- pkg/common/json_condition.go: shared entry points mirroring
  BuildSpatialCondition - ResolveJSONColumnExpr (select/sort),
  BuildJSONFilterCondition (where, full operator set; infers ::numeric for
  ordered comparisons on numeric values when no explicit cast is given),
  and the ApplySelectColumns helper.
- pkg/reflection.IsJSONColumn / pkg/spectypes.IsJSONType: disambiguate the
  dotted shorthand (data.city is JSON only when the base is a JSON column).
- pkg/common/validation.go: ColumnValidator accepts JSON tokens.
- Handlers: thread model through the filter call chains and wire the
  select/sort paths.

funcspec (raw-SQL string builder, no param binding or model) and the
FetchRowNumber raw-SQL builders are left as follow-ups, as is OpenAPI
reporting of JSON sub-field columns.
2026-09-07 17:05:52 +02:00
warkanum f841d58c59 fix(spectypes): serialize SqlTimeStamp as RFC3339 with offset
Tests / Unit Tests (push) Failing after 11s
Tests / Integration Tests (push) Failing after 23s
Build , Vet Test, and Lint / Build (push) Successful in 53s
Build , Vet Test, and Lint / Run Vet Tests (1.24.x) (push) Successful in 56s
Build , Vet Test, and Lint / Run Vet Tests (1.23.x) (push) Successful in 56s
Build , Vet Test, and Lint / Lint Code (push) Successful in 1m3s
MarshalJSON and Value now emit time.RFC3339 instead of a naive
YYYY-MM-DDTHH:MM:SS layout, so timezone offset is preserved in JSON
and DB writes. UnmarshalJSON zero-check made offset-agnostic.
2026-09-03 22:14:08 +02:00
Hein cbac47e052 fix(handler): handle JSON marshaling errors in response
Tests / Unit Tests (push) Failing after 4s
Tests / Integration Tests (push) Failing after 23s
Build , Vet Test, and Lint / Build (push) Successful in 52s
Build , Vet Test, and Lint / Run Vet Tests (1.24.x) (push) Successful in 55s
Build , Vet Test, and Lint / Run Vet Tests (1.23.x) (push) Successful in 57s
Build , Vet Test, and Lint / Lint Code (push) Successful in 58s
2026-08-31 16:12:59 +02:00
warkanum 3d4f6faa8e fix(wkb): simplify loop indexing in geometry reading functions
Tests / Integration Tests (push) Failing after 23s
Build , Vet Test, and Lint / Build (push) Successful in 4m33s
Build , Vet Test, and Lint / Run Vet Tests (1.23.x) (push) Successful in 5m13s
Build , Vet Test, and Lint / Run Vet Tests (1.24.x) (push) Successful in 5m13s
Tests / Unit Tests (push) Failing after 4s
Build , Vet Test, and Lint / Lint Code (push) Successful in 5m13s
2026-08-29 21:36:01 +02:00
warkanum f259df1258 feat(spectypes): add support for PostGIS and pgvector types
* Implement custom types: SqlGeometry, SqlGeography, SqlHalfVector, SqlSparseVector, SqlBitVector
* Add spatial filter operators and vector similarity operators
* Include metadata and OpenAPI reporting for geometry/vector column types
* Create tests for EWKB and WKT conversions
2026-08-29 21:27:42 +02:00
warkanum 798bb47e71 fix: scan relation preloads through configured model
Tests / Unit Tests (push) Failing after 46s
Build , Vet Test, and Lint / Lint Code (push) Successful in 2m48s
Tests / Integration Tests (push) Failing after 19s
Build , Vet Test, and Lint / Run Vet Tests (1.24.x) (push) Successful in 2m56s
Build , Vet Test, and Lint / Run Vet Tests (1.23.x) (push) Successful in 2m58s
Build , Vet Test, and Lint / Build (push) Failing after 1m32s
2026-08-28 22:55:44 +02:00
warkanum 105a5e1b87 fix(bun): handle panic in PreloadRelation without returning 2026-08-28 22:47:55 +02:00
warkanum a68cf83be6 fix(spec): unwrap SQL wrapper metadata types 2026-08-28 22:42:52 +02:00
warkanum dab4940ace fix(security): DatabaseAuthenticator.RefreshToken surfaces rotated refresh token and expiry
Tests / Unit Tests (push) Failing after 13s
Tests / Integration Tests (push) Failing after 27s
Build , Vet Test, and Lint / Run Vet Tests (1.24.x) (push) Successful in 33s
Build , Vet Test, and Lint / Run Vet Tests (1.23.x) (push) Successful in 35s
Build , Vet Test, and Lint / Build (push) Successful in 35s
Build , Vet Test, and Lint / Lint Code (push) Successful in 39s
RefreshToken() hardcoded LoginResponse{Token: userCtx.SessionID, ExpiresIn: 24h}
and silently discarded anything else resolvespec_refresh_token returned. An
implementation that issues its own independent, rotating refresh token (not
just reusing the session/access token as its own refresh token) has nowhere
else to put the new refresh token and real access-token expiry than
UserContext.Claims, since UserContext has no dedicated fields for either.

Now reads claims.refresh_token/claims.expires_in when present and surfaces
them into LoginResponse.RefreshToken/ExpiresIn. Implementations that don't
set these claims keep today's behavior unchanged (empty RefreshToken, 24h
ExpiresIn default) — purely additive, no breaking change.
2026-08-27 21:51:16 +02:00
Hein c7178e0a2b fix(parameters): increase default limit for requests
Tests / Integration Tests (push) Failing after 3m24s
Build , Vet Test, and Lint / Run Vet Tests (1.23.x) (push) Successful in 4m13s
Build , Vet Test, and Lint / Run Vet Tests (1.24.x) (push) Successful in 3m17s
Build , Vet Test, and Lint / Lint Code (push) Failing after 4m51s
Tests / Unit Tests (push) Failing after 5m23s
Build , Vet Test, and Lint / Build (push) Successful in 4m14s
2026-08-25 15:50:09 +02:00
warkanum 0261f121e8 fix(security): change types for template and hasBlock
Tests / Unit Tests (push) Failing after 19s
Tests / Integration Tests (push) Failing after 30s
Build , Vet Test, and Lint / Run Vet Tests (1.23.x) (push) Failing after 39s
Build , Vet Test, and Lint / Build (push) Successful in 3m47s
Build , Vet Test, and Lint / Lint Code (push) Successful in 3m45s
Build , Vet Test, and Lint / Run Vet Tests (1.24.x) (push) Successful in 3m47s
2026-08-10 20:46:40 +02:00
warkanum 93dc1008ee fix(security): unwrap userRef for non-DB providers 2026-08-10 20:34:06 +02:00
warkanum c60565e4e0 feat(resolvespec): add 'contains' operator for array overlap
* Implemented 'contains' operator using BuildArrayOverlapCondition for real array containment.
* Updated documentation to clarify differences between text-cast ILIKE and array overlap.
2026-08-10 15:01:23 +02:00
Hein 16cc7d350e fix(hooks): implement retry logic for hook registry locks
Tests / Unit Tests (push) Failing after 10s
Tests / Integration Tests (push) Failing after 13s
Build , Vet Test, and Lint / Run Vet Tests (1.23.x) (push) Successful in 1m5s
Build , Vet Test, and Lint / Run Vet Tests (1.24.x) (push) Successful in 1m3s
Build , Vet Test, and Lint / Build (push) Successful in 1m7s
Build , Vet Test, and Lint / Lint Code (push) Successful in 1m48s
* Add tryLock and tryRLock methods to manage mutex access
* Update Register, Clear, and Execute methods to handle locked state
* Log errors when registry operations fail due to locking
2026-08-04 17:55:36 +02:00
Hein a172c73ab0 fix(handler): update default sort retrieval logic
Tests / Unit Tests (push) Failing after 10s
Tests / Integration Tests (push) Failing after 13s
Build , Vet Test, and Lint / Run Vet Tests (1.23.x) (push) Successful in 2m5s
Build , Vet Test, and Lint / Lint Code (push) Failing after 5m44s
Build , Vet Test, and Lint / Build (push) Successful in 7m12s
Build , Vet Test, and Lint / Run Vet Tests (1.24.x) (push) Successful in 8m11s
2026-07-29 10:23:08 +02:00
Hein ef28959c4d test(types): add test for ResolveSortColumns with descending order 2026-07-29 10:15:52 +02:00
Hein 7c737afc5a feat(handler): implement default sort configuration for models
* Add SetDefaultSort method to configure default sort order
* Implement getDefaultSort method to retrieve configured defaults
* Update handleRead to apply default sort when none specified
* Add tests for default sort functionality
2026-07-29 10:05:24 +02:00
HeinandClaude Sonnet 5 a70e3e02d0 feat(security): complete OAuth2/OIDC spec coverage in OAuthServer
Add client_secret_basic/client_secret_post client authentication, the
client_credentials grant (RFC 6749 §4.4, backed by a synthetic
service-account user so it reuses the existing session/introspection/RLS
pipeline unchanged), RFC 9728 protected resource metadata, and OIDC
discovery + JWKS + id_token/userinfo support.

Remove plan_oauth.md, which was only meant as a working handoff doc.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
2026-07-29 09:45:09 +02:00
Hein cec8eb5c0f Merge branch 'main' of https://github.com/bitechdev/ResolveSpec
Tests / Integration Tests (push) Failing after 16s
Build , Vet Test, and Lint / Run Vet Tests (1.23.x) (push) Successful in 51s
Tests / Unit Tests (push) Failing after 11s
Build , Vet Test, and Lint / Build (push) Successful in 1m23s
Build , Vet Test, and Lint / Run Vet Tests (1.24.x) (push) Successful in 1m35s
Build , Vet Test, and Lint / Lint Code (push) Successful in 1m35s
2026-07-28 17:48:14 +02:00
Hein 06fa3198f2 feat(security): implement OAuth2 client authentication and grants
* Add client authentication methods and client_credentials grant support
* Introduce RFC 9728 Protected Resource Metadata endpoint
* Implement OIDC discovery and id_token issuance
* Update database schema for new client fields
* Add new HTTP handlers for metadata and userinfo
2026-07-28 17:48:09 +02:00
warkanum 52d3dca1fa feat(hooks): add BeforeOp hook and fix BeforeScan row-security gap
Tests / Unit Tests (push) Failing after 11s
Tests / Integration Tests (push) Failing after 15s
Build , Vet Test, and Lint / Build (push) Successful in 1m45s
Build , Vet Test, and Lint / Run Vet Tests (1.24.x) (push) Successful in 2m0s
Build , Vet Test, and Lint / Run Vet Tests (1.23.x) (push) Successful in 2m0s
Build , Vet Test, and Lint / Lint Code (push) Successful in 2m4s
Adds a BeforeOp HookType across resolvespec, restheadspec, websocketspec,
mqttspec, and funcspec that fires before every SQL operation (read,
create, update, delete, scan/query) via a new ExecuteBeforeOp helper.

Also closes a row-level-security gap: resolvespec registered a BeforeScan
hook for ApplyRowSecurity but never fired it, and websocketspec/mqttspec
had no BeforeScan hook point at all, so row security was never applied
to their queries. BeforeScan now fires right before the actual scan in
all three, with the (possibly hook-modified) query used for execution.
2026-07-25 11:39:31 +02:00
Hein 873e8925d4 fix(handler): ensure target ID is provided for updates
Tests / Unit Tests (push) Failing after 9s
Tests / Integration Tests (push) Failing after 12s
Build , Vet Test, and Lint / Run Vet Tests (1.23.x) (push) Successful in 48s
Build , Vet Test, and Lint / Build (push) Successful in 1m18s
Build , Vet Test, and Lint / Run Vet Tests (1.24.x) (push) Successful in 1m27s
Build , Vet Test, and Lint / Lint Code (push) Successful in 1m28s
2026-07-24 16:35:22 +02:00
Hein b23916048a fix: ordering in before hook 2026-07-24 15:56:23 +02:00
Hein 47708fc87a fix(security): run RLS-scoping hooks in the same transaction as their queries
Tests / Unit Tests (push) Failing after 12s
Tests / Integration Tests (push) Failing after 16s
Build , Vet Test, and Lint / Run Vet Tests (1.23.x) (push) Successful in 51s
Build , Vet Test, and Lint / Build (push) Successful in 1m24s
Build , Vet Test, and Lint / Run Vet Tests (1.24.x) (push) Successful in 1m31s
Build , Vet Test, and Lint / Lint Code (push) Successful in 2m4s
BeforeRead/BeforeCreate hooks (used to set session-scoped RLS GUCs) were
firing against the pooled db handle while the actual queries ran as
separate calls to the same pool. Under connection pooling these could
land on different physical connections, silently bypassing row-level
security on creates and reads. handleUpdate already did this correctly;
handleRead/handleCreate in both resolvespec and restheadspec now wrap
hook execution and queries in a single RunInTransaction call.
2026-07-24 12:40:27 +02:00
Hein a85e572732 fix(hooks): reset Tx to pooled connection for post-commit hook calls
Tests / Unit Tests (push) Failing after 2m40s
Build , Vet Test, and Lint / Run Vet Tests (1.23.x) (push) Successful in 2m9s
Tests / Integration Tests (push) Failing after 2m47s
Build , Vet Test, and Lint / Build (push) Successful in 5m56s
Build , Vet Test, and Lint / Lint Code (push) Failing after 6m37s
Build , Vet Test, and Lint / Run Vet Tests (1.24.x) (push) Successful in 6m48s
BeforeScan (restheadspec handleUpdate) and BeforeResponse (funcspec
list/single query handlers) fire after RunInTransaction commits, but
hookCtx.Tx still pointed at the now-dead transaction. Any hook that
executed a query against Tx (e.g. setUserViaContext) failed with
"sql: transaction has already been committed or rolled back".
2026-07-20 13:45:13 +02:00
Hein 598fd687f6 feat(security): add GetUserRef method for opaque user identifiers
Tests / Integration Tests (push) Failing after 1s
Build , Vet Test, and Lint / Run Vet Tests (1.24.x) (push) Successful in 2m6s
Build , Vet Test, and Lint / Lint Code (push) Successful in 2m21s
Tests / Unit Tests (push) Failing after 21s
Build , Vet Test, and Lint / Build (push) Successful in 50s
Build , Vet Test, and Lint / Run Vet Tests (1.23.x) (push) Successful in 54s
2026-07-10 13:47:32 +02:00
Hein eee83f9dc6 feat(security): add query mode handling for database operations
Tests / Integration Tests (push) Failing after 1s
Tests / Unit Tests (push) Failing after 24s
Build , Vet Test, and Lint / Run Vet Tests (1.23.x) (push) Successful in 59s
Build , Vet Test, and Lint / Run Vet Tests (1.24.x) (push) Successful in 58s
Build , Vet Test, and Lint / Build (push) Successful in 4m37s
Build , Vet Test, and Lint / Lint Code (push) Failing after 5m35s
* Introduced QueryMode to select between stored procedure and direct SQL execution.
* Implemented dbCapability to probe for stored procedure existence.
* Added table names configuration for direct SQL operations.
* Updated DatabaseTwoFactorProvider to support query mode and table names.
* Implemented direct SQL methods mirroring stored procedures for TOTP operations.
* Added tests for query mode logic and table names validation.
2026-07-07 15:29:29 +02:00
Hein 8a06aacfb2 fix(cors): update CORS headers handling for requests
Tests / Integration Tests (push) Failing after 1s
Tests / Unit Tests (push) Failing after 22s
Build , Vet Test, and Lint / Build (push) Successful in 1m1s
Build , Vet Test, and Lint / Lint Code (push) Successful in 1m19s
Build , Vet Test, and Lint / Run Vet Tests (1.23.x) (push) Successful in 1m35s
Build , Vet Test, and Lint / Run Vet Tests (1.24.x) (push) Successful in 1m36s
* Reflect request origin for Access-Control-Allow-Origin
* Set Vary header for caching based on origin
* Allow specific headers from preflight requests
* Enable credentials only for specific origins
2026-07-01 12:27:39 +02:00
Hein 705c4f8001 fix(manager): ensure HTTP1 is set when HTTP2 is disabled
Tests / Integration Tests (push) Failing after 1s
Tests / Unit Tests (push) Failing after 1m41s
Build , Vet Test, and Lint / Run Vet Tests (1.23.x) (push) Successful in 3m50s
Build , Vet Test, and Lint / Run Vet Tests (1.24.x) (push) Successful in 3m53s
Build , Vet Test, and Lint / Build (push) Successful in 3m55s
Build , Vet Test, and Lint / Lint Code (push) Successful in 4m3s
2026-06-30 13:54:56 +02:00
Hein d648614611 feat(config): add PanicHandler to Config for custom recovery 2026-06-30 13:49:51 +02:00
Hein 3f86eb0f06 feat(config): add HTTP2 field to ServerInstanceConfig and align with server.Config 2026-06-30 13:36:06 +02:00
Hein 3dac55cb19 fix: Set http2 based on prop 2026-06-30 13:29:55 +02:00
Hein bbb2c6d127 feat(server): add HTTP2 support in server configuration 2026-06-30 11:33:31 +02:00
Hein 3fec7b1a90 fix(handler): update Content-Range headers for API response
Build , Vet Test, and Lint / Lint Code (push) Failing after 0s
Build , Vet Test, and Lint / Run Vet Tests (1.23.x) (push) Successful in 48s
Build , Vet Test, and Lint / Build (push) Failing after 0s
Build , Vet Test, and Lint / Run Vet Tests (1.24.x) (push) Failing after 1s
Tests / Unit Tests (push) Failing after 1s
Tests / Integration Tests (push) Failing after 0s
* change Content-Range format to include 'items'
* add X-Api-Range-From and X-Api-Modelname headers
* add X-Api-Range-Etotal header for total filtered items
2026-06-24 10:02:54 +02:00
Hein 910390f62d fix(headers): correct order of limit and offset parsing 2026-06-24 09:47:14 +02:00
Hein b9bed67bd7 feat(security): add program user ID and table to user context
Build , Vet Test, and Lint / Lint Code (push) Failing after 0s
Build , Vet Test, and Lint / Run Vet Tests (1.24.x) (push) Failing after 0s
Build , Vet Test, and Lint / Build (push) Failing after 1s
Tests / Unit Tests (push) Failing after 0s
Tests / Integration Tests (push) Failing after 1s
Build , Vet Test, and Lint / Run Vet Tests (1.23.x) (push) Failing after 14m57s
2026-06-23 10:18:22 +02:00
Hein 11ef16f75a fix(sql_helpers): adjust parenthesis nesting depth comment 2026-06-23 09:41:40 +02:00
Hein 48b72a7631 fix(sql_helpers): enhance splitByAND to handle BETWEEN and quotes
* Add support for BETWEEN-aware AND detection
* Ensure AND inside single-quoted strings does not cause splits
* Update tests to cover new BETWEEN and quote scenarios
2026-06-23 09:41:27 +02:00
Hein 4c512acf25 test(function_api): add test for x-detailapi header response 2026-06-23 08:53:33 +02:00
Hein 07a402634e fix(function_api): enhance detail format with table metadata
* include table name and prefix in response
* add field metadata extraction for raw SQL results
2026-06-23 08:50:29 +02:00
Hein 0e8f8925c6 fix(reflection): replace reflect.Ptr with reflect.Pointer
Build , Vet Test, and Lint / Run Vet Tests (1.23.x) (push) Failing after 0s
Build , Vet Test, and Lint / Run Vet Tests (1.24.x) (push) Failing after 1s
Build , Vet Test, and Lint / Lint Code (push) Failing after 0s
Build , Vet Test, and Lint / Build (push) Failing after 0s
Tests / Unit Tests (push) Failing after 1s
Tests / Integration Tests (push) Failing after 1s
* Updated all instances of reflect.Ptr to reflect.Pointer for consistency in type checking.
2026-06-22 16:40:07 +02:00
Hein 5a359a160b fix(handler): update sendFormattedResponse to include table name and model 2026-06-22 16:38:21 +02:00
Hein a2799fa224 fix(handler): re-fetch records to capture DB-generated values
Build , Vet Test, and Lint / Run Vet Tests (1.23.x) (push) Has been cancelled
Build , Vet Test, and Lint / Run Vet Tests (1.24.x) (push) Has been cancelled
Build , Vet Test, and Lint / Lint Code (push) Has been cancelled
Build , Vet Test, and Lint / Build (push) Has been cancelled
Tests / Unit Tests (push) Has been cancelled
Tests / Integration Tests (push) Has been cancelled
2026-06-12 16:28:51 +02:00
Hein 1419542650 fix(handler): re-fetch records to capture DB-generated changes 2026-06-12 13:37:07 +02:00
Hein c120b49529 fix(router): prevent HTML escaping in JSON responses
Build , Vet Test, and Lint / Run Vet Tests (1.24.x) (push) Has been cancelled
Build , Vet Test, and Lint / Lint Code (push) Has been cancelled
Build , Vet Test, and Lint / Build (push) Has been cancelled
Tests / Integration Tests (push) Has been cancelled
Tests / Unit Tests (push) Has been cancelled
Build , Vet Test, and Lint / Run Vet Tests (1.23.x) (push) Has been cancelled
fix(sql_helpers): avoid prefix extraction in subqueries
2026-06-08 15:13:58 +02:00
Hein 66348dac97 test(handler): add tests for valid nested request verbs 2026-06-08 09:06:29 +02:00
Hein a87cd18b1b fix(handler): validate nested request structure for relations
* added checks for valid _request values in single and multiple relations
* introduced isValidNestedRequest function to encapsulate validation logic
fix(crud): expand operation handling for nested CUD
* added "add" to insert operations and "modify" to update operations
* included "remove" in delete operations
2026-06-08 09:02:29 +02:00
Hein 29449c93d5 fix(test): add tests for asymmetric join column handling
Build , Vet Test, and Lint / Run Vet Tests (1.23.x) (push) Has been cancelled
Build , Vet Test, and Lint / Run Vet Tests (1.24.x) (push) Has been cancelled
Build , Vet Test, and Lint / Lint Code (push) Has been cancelled
Build , Vet Test, and Lint / Build (push) Has been cancelled
Tests / Unit Tests (push) Has been cancelled
Tests / Integration Tests (push) Has been cancelled
2026-06-07 19:13:59 +02:00
Hein 3b6e5c75be fix(handler): update foreign key field resolution logic
Build , Vet Test, and Lint / Run Vet Tests (1.23.x) (push) Has been cancelled
Build , Vet Test, and Lint / Run Vet Tests (1.24.x) (push) Has been cancelled
Build , Vet Test, and Lint / Lint Code (push) Has been cancelled
Build , Vet Test, and Lint / Build (push) Has been cancelled
Tests / Unit Tests (push) Has been cancelled
Tests / Integration Tests (push) Has been cancelled
* Adjust foreign key field name selection for has-many/has-one relationships
* Improve logging to clarify foreign key and child field usage
2026-06-07 14:20:55 +02:00
Hein 549ccb8468 fix(handler): fetch updated records after transaction commits
Build , Vet Test, and Lint / Run Vet Tests (1.23.x) (push) Has been cancelled
Build , Vet Test, and Lint / Run Vet Tests (1.24.x) (push) Has been cancelled
Build , Vet Test, and Lint / Lint Code (push) Has been cancelled
Build , Vet Test, and Lint / Build (push) Has been cancelled
Tests / Unit Tests (push) Has been cancelled
Tests / Integration Tests (push) Has been cancelled
* Update selection queries to use model columns
* Ensure updated records are fetched and returned in responses
2026-06-05 11:12:04 +02:00
Hein 1af9c76337 fix(handler): fetch updated record after transaction commits
Tests / Unit Tests (push) Has been cancelled
Tests / Integration Tests (push) Has been cancelled
Build , Vet Test, and Lint / Run Vet Tests (1.23.x) (push) Has been cancelled
Build , Vet Test, and Lint / Run Vet Tests (1.24.x) (push) Has been cancelled
Build , Vet Test, and Lint / Lint Code (push) Has been cancelled
Build , Vet Test, and Lint / Build (push) Has been cancelled
2026-06-04 18:23:18 +02:00
Hein 938a2ef3d9 fix(staticweb): add fallback for extensionless file paths
Build , Vet Test, and Lint / Run Vet Tests (1.24.x) (push) Failing after 6s
Tests / Integration Tests (push) Failing after 13m59s
Tests / Unit Tests (push) Failing after 14m11s
Build , Vet Test, and Lint / Build (push) Failing after 14m21s
Build , Vet Test, and Lint / Lint Code (push) Failing after 14m31s
Build , Vet Test, and Lint / Run Vet Tests (1.23.x) (push) Failing after 14m45s
2026-05-27 18:41:43 +02:00
Hein 69cc3e2839 fix(db): update Returning method to accept multiple columns 2026-05-27 14:11:20 +02:00
Hein 4018af0636 fix(validation): enhance filter logic for column validation
* adjust handling of "all" filter to consider filtered columns
fix(function_api): improve variable substitution in SQL queries
* add safeSubstituteVar for context-aware value sanitization
2026-05-27 12:17:31 +02:00
Hein c4e79d6950 fix(validation): use strings.EqualFold for case-insensitive comparison 2026-05-27 12:07:08 +02:00
Hein 982a0e62ac fix(validation): add Columns method to retrieve valid column names 2026-05-27 12:06:46 +02:00
Hein 5d459c95a7 fix(headers): reorder import statements for clarity 2026-05-27 11:28:39 +02:00
Hein e9f7726e43 fix(headers): sort combined parameters before processing 2026-05-27 11:28:22 +02:00
Hein 3d2251317a fix(headers): remove unused utf8 validation in DecodeParam
Build , Vet Test, and Lint / Run Vet Tests (1.24.x) (push) Failing after 1s
Build , Vet Test, and Lint / Run Vet Tests (1.23.x) (push) Failing after 1s
Build , Vet Test, and Lint / Lint Code (push) Failing after 1s
Build , Vet Test, and Lint / Build (push) Failing after 1s
Tests / Unit Tests (push) Failing after 0s
Tests / Integration Tests (push) Failing after 1s
2026-05-26 10:31:34 +02:00
Hein 1ce0ab1ab4 fix(validation): improve preload column validation logic
Build , Vet Test, and Lint / Run Vet Tests (1.23.x) (push) Failing after -35m6s
Build , Vet Test, and Lint / Run Vet Tests (1.24.x) (push) Failing after -35m6s
Build , Vet Test, and Lint / Lint Code (push) Failing after -35m6s
Build , Vet Test, and Lint / Build (push) Failing after -35m6s
Tests / Unit Tests (push) Failing after -35m7s
Tests / Integration Tests (push) Failing after -35m7s
2026-05-21 20:18:01 +02:00
Hein 1f9b230f7f fix(validation): improve preload column validation logic
* Use related model's validator for filtering preload columns
* Ensure valid columns are checked against the correct validator
2026-05-21 20:16:53 +02:00
Hein c42c6b28e3 fix(errortracking): update error context handling in Sentry
* Replace event.Extra with event.Contexts["extra"] for error and message captures.
* Consolidate extra context handling in CapturePanic method.
2026-05-21 14:03:01 +02:00
Hein 57e7503389 feat(staticweb): enhance fallback logic for extensionless paths
Build , Vet Test, and Lint / Run Vet Tests (1.23.x) (push) Failing after -35m7s
Build , Vet Test, and Lint / Run Vet Tests (1.24.x) (push) Failing after -35m7s
Build , Vet Test, and Lint / Lint Code (push) Failing after -35m7s
Build , Vet Test, and Lint / Build (push) Failing after -35m7s
Tests / Unit Tests (push) Failing after -35m8s
Tests / Integration Tests (push) Failing after -35m8s
* Added support for serving index.html for extensionless paths
* Updated isStaticAsset to exclude paths without extensions from static asset checks
2026-05-21 13:30:39 +02:00
Hein 0308644075 feat(auth): add authenticate callback for fallback logic
* Implement SetAuthenticateCallback in authenticators
* Update Authenticate methods to use callback on failure
2026-05-21 11:27:51 +02:00
Hein e5984f5205 feat(security): add cookie support for login and logout methods
* Implement LoginWithCookie and LogoutWithCookie in stubAuthenticator, mockAuth, mockSecurityProvider, and MockAuthenticator
* Update tests to use cookie-based authentication
2026-05-21 09:51:30 +02:00
Hein 76909ae869 feat(security): add cookie support for login and logout methods
* Implement LoginWithCookie and LogoutWithCookie in CompositeSecurityProvider
* Update Authenticator interface to include cookie methods
* Add cookie support in HeaderAuthenticator and JWTAuthenticator
2026-05-21 09:48:46 +02:00
Hein c90c2984ac feat(security): add cookie session support to DatabaseAuthenticator
* Introduce enableCookieSession option for session management
* Implement LoginWithCookie and LogoutWithCookie methods
* Update Authenticate method to support session token from cookie
2026-05-21 09:14:50 +02:00
Hein 1ab4ae33e7 feat(security): implement ChainAuthenticator for sequential authentication 2026-05-21 08:35:39 +02:00
Hein 905457964c fix(restheadspec): remove redundant column selection in query 2026-05-21 08:34:09 +02:00
Hein c42d09238f fix: better error detail for failed sql
Build , Vet Test, and Lint / Run Vet Tests (1.24.x) (push) Failing after -35m9s
Build , Vet Test, and Lint / Build (push) Failing after -35m9s
Tests / Unit Tests (push) Failing after -35m10s
Build , Vet Test, and Lint / Run Vet Tests (1.23.x) (push) Failing after -35m9s
Build , Vet Test, and Lint / Lint Code (push) Failing after -35m9s
Tests / Integration Tests (push) Failing after -35m10s
2026-05-20 13:06:26 +02:00
Hein 0647a88aba fix(restheadspec): qualify primary key and sort columns with table alias 2026-05-20 12:41:16 +02:00
Hein 3d2e11eeed fix(restheadspec): always respond 200 OK regardless of result count in sendFormattedResponse
Build , Vet Test, and Lint / Run Vet Tests (1.24.x) (push) Successful in -33m27s
Build , Vet Test, and Lint / Run Vet Tests (1.23.x) (push) Successful in -32m51s
Build , Vet Test, and Lint / Lint Code (push) Failing after -33m3s
Build , Vet Test, and Lint / Build (push) Successful in -33m10s
Tests / Unit Tests (push) Successful in -33m58s
Tests / Integration Tests (push) Failing after -34m20s
2026-05-19 09:46:25 +02:00
Hein 4493bfa40f feat(reflection): add IsEmptyValue helper; guard CUD ops against missing PK
Add reflection.IsEmptyValue to detect nil, empty string, and zero numbers.
Use it in recursive CUD processing to skip update/delete when the primary
key is absent, logging a warning instead of proceeding with an invalid operation.
2026-05-19 09:14:19 +02:00
Hein b157379ff8 fix(restheadspec): return 200 OK with empty body instead of 204 on zero results
Frontend clients are sensitive to 204 No Content responses; always return 200
with an empty array/object and rely on X-No-Data-Found header to signal absence
of records.

Also treat "change" as an alias for "update" in recursive CUD processing.
2026-05-19 08:56:11 +02:00
Hein 52752d9c8b fix(bun): adjust field alignment in BunSelectQuery struct
Build , Vet Test, and Lint / Run Vet Tests (1.24.x) (push) Successful in -33m53s
Build , Vet Test, and Lint / Run Vet Tests (1.23.x) (push) Successful in -33m29s
Build , Vet Test, and Lint / Lint Code (push) Failing after -33m33s
Build , Vet Test, and Lint / Build (push) Successful in -33m41s
Tests / Unit Tests (push) Successful in -34m28s
Tests / Integration Tests (push) Failing after -34m36s
2026-05-18 17:12:32 +02:00
Hein baca5ad29e fix(bun): add relation alias handling for separate-query preloads
* implement preloadRelationAlias to rewrite WHERE conditions
* update Where method to handle relation alias in queries
2026-05-18 17:12:21 +02:00
Hein 53ab22ce02 fix(nestedCUD): handle error in processChildRelations gracefully 2026-05-18 16:14:24 +02:00
Hein 09a3dc92b9 fix(restheadspec): normalize empty results to objects instead of arrays 2026-05-18 14:37:46 +02:00
Hein 6590cd789a fix(nestedCUD): re-select rows after insert/update for accurate state
* Ensure result.Data reflects DB-generated defaults after insert.
* Update result.Data with current DB state after update.
2026-05-18 13:10:13 +02:00
Hein 4244e838b1 fix(reflection): enhance GetForeignKeyColumn logic for self-referential models
* Add support for self-referential models in GetForeignKeyColumn
* Update comments for clarity on foreign key resolution strategies
* Introduce selfRefItem struct for testing self-referential behavior
2026-05-18 13:03:07 +02:00
Hein c42fa11c1a fix(reflection): update GetForeignKeyColumn to return multiple columns
* Change return type to []string for composite keys
* Adjust related logic in injectForeignKeys method
* Update tests to validate new behavior for composite foreign keys
2026-05-18 12:39:06 +02:00
Hein 85bb0f7874 fix(funcspec): update meta variable replacement in SQL query 2026-05-18 12:13:06 +02:00
Hein cd65946191 fix(database): add Scan method to insert query interfaces
* Implement Scan method for BunInsertQuery, GormInsertQuery, and PgSQLInsertQuery
* Update mock implementations to support Scan method
* Introduce GetForeignKeyColumn utility for foreign key resolution
* Add tests for GetForeignKeyColumn functionality
2026-05-18 12:04:50 +02:00
Hein cb416d49c4 fix(headers): handle decoding errors in header values
Build , Vet Test, and Lint / Run Vet Tests (1.24.x) (push) Successful in -33m58s
Build , Vet Test, and Lint / Run Vet Tests (1.23.x) (push) Successful in -33m22s
Build , Vet Test, and Lint / Lint Code (push) Failing after -33m34s
Build , Vet Test, and Lint / Build (push) Successful in -33m45s
Tests / Unit Tests (push) Failing after -34m38s
Tests / Integration Tests (push) Failing after -34m48s
* return original value if decoding fails
* decode base64 strings when appropriate
2026-05-15 16:59:06 +02:00
Hein cb921f2c5e fix(websocketspec): add transaction access to HookContext 2026-05-15 14:59:34 +02:00
Hein 1ebe0d7ac3 fix(funcspec): refine filter application logic for SQL queries
* update filter checks to only consider SELECT list
* add test for function parameters not matching filters
2026-05-15 14:28:12 +02:00
Hein ae9e06c98b fix(sql_helpers): strip empty RHS conditions from SQL strings
* Add regex patterns to identify and remove empty comparisons
* Implement tests for stripping empty RHS conditions
fix(handler): prevent duplicate JOIN aliases from preload
* Skip custom SQL JOINs if alias already provided by preload
* Split multiple JOIN clauses for individual alias handling
2026-05-15 13:35:24 +02:00
Hein 2ae4d07544 fix(funcspec): remove AllowQueryParamFilters and related logic
Build , Vet Test, and Lint / Run Vet Tests (1.23.x) (push) Successful in -26m26s
Build , Vet Test, and Lint / Build (push) Successful in -34m6s
Build , Vet Test, and Lint / Lint Code (push) Successful in -32m6s
Tests / Integration Tests (push) Failing after -34m50s
Tests / Unit Tests (push) Successful in -30m42s
Build , Vet Test, and Lint / Run Vet Tests (1.24.x) (push) Successful in -26m44s
* Simplify SqlQueryOptions by removing AllowQueryParamFilters
* Update mergeQueryParams to avoid applying filters for JSON arguments
* Add tests for sqlStripStringLiterals and query param handling
2026-05-15 09:25:55 +02:00
Hein 49639b6c19 fix(funcspec): add support for query param filters
* Introduced AllowQueryParamFilters option in SqlQueryOptions
* Implemented applyQueryParamFilters method to handle field filters
2026-05-15 09:07:21 +02:00
Hein 8733176cba fix(funcspec): enhance quote detection for parameters 2026-05-15 08:26:59 +02:00
Hein bce27f7ed2 fix: 🐛 Fixed array to slice array resolution on reflection GetRelationType
Build , Vet Test, and Lint / Run Vet Tests (1.23.x) (push) Failing after -35m20s
Build , Vet Test, and Lint / Run Vet Tests (1.24.x) (push) Failing after -35m20s
Build , Vet Test, and Lint / Lint Code (push) Failing after -35m20s
Build , Vet Test, and Lint / Build (push) Failing after -35m20s
Tests / Unit Tests (push) Failing after -35m21s
Tests / Integration Tests (push) Failing after -35m21s
2026-05-11 14:24:25 +02:00
Hein 987a2a7faf fix(db): convert slices to PostgreSQL array literals in queries
Build , Vet Test, and Lint / Run Vet Tests (1.24.x) (push) Successful in -32m17s
Build , Vet Test, and Lint / Run Vet Tests (1.23.x) (push) Successful in -31m49s
Build , Vet Test, and Lint / Build (push) Successful in -31m53s
Build , Vet Test, and Lint / Lint Code (push) Successful in -31m11s
Tests / Unit Tests (push) Successful in -32m31s
Tests / Integration Tests (push) Failing after -32m46s
2026-05-07 14:33:35 +02:00
warkanum 157788b73b fix(todo): document issue with GormResult.LastInsertId() not returning correct ID
Build , Vet Test, and Lint / Run Vet Tests (1.24.x) (push) Successful in -32m18s
Build , Vet Test, and Lint / Run Vet Tests (1.23.x) (push) Successful in -31m19s
Build , Vet Test, and Lint / Lint Code (push) Successful in -31m5s
Build , Vet Test, and Lint / Build (push) Successful in -31m53s
Tests / Integration Tests (push) Failing after -32m52s
Tests / Unit Tests (push) Successful in -32m36s
2026-05-05 09:52:31 +02:00
Hein fb051b5577 fix(spectypes): correct quoting logic in formatPostgresStringArray
Build , Vet Test, and Lint / Run Vet Tests (1.23.x) (push) Failing after -33m2s
Build , Vet Test, and Lint / Run Vet Tests (1.24.x) (push) Successful in -28m38s
Build , Vet Test, and Lint / Lint Code (push) Successful in -27m26s
Build , Vet Test, and Lint / Build (push) Successful in -29m3s
Tests / Integration Tests (push) Failing after -33m9s
Tests / Unit Tests (push) Successful in -30m2s
2026-04-30 15:38:21 +02:00
Hein cc9c4337fd feat(spectypes): add PostgreSQL array types and parsing functions 2026-04-30 15:37:33 +02:00
Hein 0aaeff63a2 fix(db): guard against non-existent relations in preload
Build , Vet Test, and Lint / Run Vet Tests (1.24.x) (push) Successful in -33m2s
Build , Vet Test, and Lint / Run Vet Tests (1.23.x) (push) Successful in -32m35s
Build , Vet Test, and Lint / Lint Code (push) Successful in -32m33s
Build , Vet Test, and Lint / Build (push) Successful in -32m50s
Tests / Integration Tests (push) Failing after -33m35s
Tests / Unit Tests (push) Successful in -33m22s
2026-04-20 17:15:33 +02:00
Hein 325769be4e feat(reflection): add support for nested struct mapping
Build , Vet Test, and Lint / Run Vet Tests (1.24.x) (push) Successful in -33m9s
Build , Vet Test, and Lint / Run Vet Tests (1.23.x) (push) Successful in -32m37s
Build , Vet Test, and Lint / Build (push) Successful in -32m41s
Build , Vet Test, and Lint / Lint Code (push) Successful in -31m55s
Tests / Unit Tests (push) Successful in -33m29s
Tests / Integration Tests (push) Failing after -33m42s
2026-04-16 13:45:46 +02:00
warkanum f79a400772 feat(security): add self-service password reset functionality
Build , Vet Test, and Lint / Run Vet Tests (1.24.x) (push) Successful in -33m14s
Build , Vet Test, and Lint / Run Vet Tests (1.23.x) (push) Successful in -32m42s
Build , Vet Test, and Lint / Build (push) Successful in -32m59s
Build , Vet Test, and Lint / Lint Code (push) Successful in -32m26s
Tests / Integration Tests (push) Failing after -33m40s
Tests / Unit Tests (push) Successful in -33m35s
* Implement password reset request and completion procedures
* Update database schema for password reset tokens
* Add new request and response types for password reset
2026-04-15 21:46:33 +02:00
Hein aef1f96c10 fix(db): cast columns to text for LIKE/ILIKE queries
Build , Vet Test, and Lint / Run Vet Tests (1.23.x) (push) Successful in -29m15s
Build , Vet Test, and Lint / Run Vet Tests (1.24.x) (push) Successful in -29m7s
Build , Vet Test, and Lint / Build (push) Successful in -32m31s
Build , Vet Test, and Lint / Lint Code (push) Successful in -31m40s
Tests / Integration Tests (push) Failing after -33m31s
Tests / Unit Tests (push) Successful in -31m4s
2026-04-13 14:05:17 +02:00
Hein 354ed2a8dc feat(db): add fallback metric entity handling for unknown targets
Build , Vet Test, and Lint / Run Vet Tests (1.24.x) (push) Successful in -30m5s
Build , Vet Test, and Lint / Run Vet Tests (1.23.x) (push) Successful in -29m35s
Build , Vet Test, and Lint / Lint Code (push) Successful in -29m19s
Build , Vet Test, and Lint / Build (push) Successful in -29m43s
Tests / Integration Tests (push) Failing after -30m34s
Tests / Unit Tests (push) Successful in -30m8s
* implement fallbackMetricEntityFromQuery for query sanitization
* add tests for fallback metric entity and sanitization logic
2026-04-10 16:00:22 +02:00
Hein dfb63c3328 refactor(db): remove metrics enabling methods from adapters 2026-04-10 14:13:15 +02:00
Hein e8d0ab28c3 feat(db): add query metrics tracking for database operations
* Introduced metrics tracking for SELECT, INSERT, UPDATE, and DELETE operations.
* Added methods to enable or disable metrics on the PgSQLAdapter.
* Created a new query_metrics.go file to handle metrics recording logic.
* Updated interfaces and implementations to support schema and entity tracking.
* Added tests to verify metrics recording functionality.
2026-04-10 13:51:46 +02:00
Hein 4fc25c60ae fix(db): correct connection pool assignment in GORM adapter
Build , Vet Test, and Lint / Run Vet Tests (1.24.x) (push) Successful in -29m43s
Build , Vet Test, and Lint / Run Vet Tests (1.23.x) (push) Successful in -29m6s
Build , Vet Test, and Lint / Lint Code (push) Successful in -29m11s
Build , Vet Test, and Lint / Build (push) Successful in -29m33s
Tests / Unit Tests (push) Successful in -30m4s
Tests / Integration Tests (push) Failing after -30m13s
2026-04-10 11:20:44 +02:00
Hein 16a960d973 feat(db): add reconnect logic for database adapters
* Implement reconnect functionality in GormAdapter and other database adapters.
* Introduce a DBFactory to handle reconnections.
* Update health check logic to skip reconnects for transient failures.
* Add tests for reconnect behavior in DatabaseAuthenticator.
2026-04-10 11:18:39 +02:00
Hein 2afee9d238 fix(db): handle database reconnection in transactions 2026-04-10 08:42:41 +02:00
Hein Puth (Warkanum) 1e89124c97 Merge pull request #18 from bitechdev/feature-auth-mcp
Build , Vet Test, and Lint / Run Vet Tests (1.24.x) (push) Successful in -30m1s
Build , Vet Test, and Lint / Run Vet Tests (1.23.x) (push) Successful in -29m29s
Build , Vet Test, and Lint / Build (push) Successful in -29m37s
Build , Vet Test, and Lint / Lint Code (push) Successful in -28m58s
Tests / Unit Tests (push) Successful in -30m23s
Tests / Integration Tests (push) Failing after -30m31s
feat(security): implement OAuth2 authorization server with database s…
2026-04-09 16:18:18 +02:00
copilot-swe-agent[bot]andwarkanum ca0545e144 fix(security): address validation review comments - mutex safety and issuer normalization
Agent-Logs-Url: https://github.com/bitechdev/ResolveSpec/sessions/e886b781-c910-425f-aa6f-06d13c46dcc7

Co-authored-by: warkanum <208308+warkanum@users.noreply.github.com>
2026-04-09 14:07:45 +00:00
copilot-swe-agent[bot]andwarkanum 850ad2b2ab fix(security): address all OAuth2 PR review issues
Agent-Logs-Url: https://github.com/bitechdev/ResolveSpec/sessions/e886b781-c910-425f-aa6f-06d13c46dcc7

Co-authored-by: warkanum <208308+warkanum@users.noreply.github.com>
2026-04-09 14:04:53 +00:00
Hein 2a2e33da0c Merge branch 'main' of https://github.com/bitechdev/ResolveSpec into feature-auth-mcp 2026-04-09 15:52:26 +02:00
Hein Puth (Warkanum) 17808a8121 Merge pull request #19 from bitechdev/feature-keystore
Feature keystore
2026-04-09 15:50:36 +02:00
Hein 134ff85c59 Merge branch 'main' of https://github.com/bitechdev/ResolveSpec into feature-keystore 2026-04-09 15:47:54 +02:00
Hein bacddc58a6 style(recursive_crud): remove unnecessary blank line 2026-04-09 15:37:13 +02:00
Hein f1ad83d966 feat(reflection): add JSON to DB column name mapping functions
* Implement BuildJSONToDBColumnMap for translating JSON keys to DB column names
* Enhance GetColumnName to extract column names with priority
* Update filterValidFields to utilize new mapping for improved data handling
* Fix TestToSnakeCase expected values for consistency
2026-04-09 15:36:52 +02:00
Hein 79a3912f93 fix(db): improve database connection handling and reconnection logic
* Added a database factory function to allow reconnection when the database is closed.
* Implemented mutex locks for safe concurrent access to the database connection.
* Updated all database query methods to handle reconnection attempts on closed connections.
* Enhanced error handling for database operations across multiple providers.
2026-04-09 09:19:28 +02:00
warkanum 6502b55797 feat(security): implement OAuth2 authorization server with database support
- Add OAuthServer for handling OAuth2 flows including authorization, token exchange, and client registration.
- Introduce DatabaseAuthenticator for persisting clients and authorization codes.
- Implement SQL procedures for client registration, code saving, and token introspection.
- Support for external OAuth2 providers and PKCE (Proof Key for Code Exchange).
2026-04-07 22:56:05 +02:00
warkanum aa095d6bfd fix(tests): replace panic with log.Fatal for better error handling
Build , Vet Test, and Lint / Run Vet Tests (1.23.x) (push) Successful in -29m52s
Build , Vet Test, and Lint / Build (push) Successful in -29m52s
Tests / Integration Tests (push) Failing after -30m46s
Tests / Unit Tests (push) Successful in -28m51s
Build , Vet Test, and Lint / Lint Code (push) Failing after -29m23s
Build , Vet Test, and Lint / Run Vet Tests (1.24.x) (push) Successful in -30m17s
2026-04-07 20:38:22 +02:00
warkanum ea5bb38ee4 feat(handler): update to use static base path for SSE server 2026-04-07 20:03:43 +02:00
warkanum c2e2c9b873 feat(transport): add streamable HTTP transport for MCP 2026-04-07 19:52:38 +02:00
warkanum 4adf94fe37 feat(go.mod): add mcp-go dependency for enhanced functionality 2026-04-07 19:09:51 +02:00
Hein a9bf08f58b feat(security): implement keystore for user authentication keys
* Add ConfigKeyStore for in-memory key management
* Introduce DatabaseKeyStore for PostgreSQL-backed key storage
* Create KeyStoreAuthenticator for API key validation
* Define SQL procedures for key management in PostgreSQL
* Document keystore functionality and usage in KEYSTORE.md
2026-04-07 17:09:17 +02:00
Hein 405a04a192 feat(security): integrate security hooks for access control
Build , Vet Test, and Lint / Lint Code (push) Failing after -30m6s
Tests / Unit Tests (push) Successful in -30m22s
Tests / Integration Tests (push) Failing after -30m41s
Build , Vet Test, and Lint / Run Vet Tests (1.24.x) (push) Successful in -30m3s
Build , Vet Test, and Lint / Run Vet Tests (1.23.x) (push) Successful in -29m36s
Build , Vet Test, and Lint / Build (push) Successful in -29m58s
* Add security hooks for per-entity operation rules and row/column-level security.
* Implement annotation tool for storing and retrieving freeform annotations.
* Enhance handler to support model registration with access rules.
2026-04-07 15:53:12 +02:00
Hein c1b16d363a feat(db): add DB method to sqlConnection and mongoConnection
Build , Vet Test, and Lint / Run Vet Tests (1.24.x) (push) Successful in -30m22s
Build , Vet Test, and Lint / Run Vet Tests (1.23.x) (push) Successful in -29m59s
Build , Vet Test, and Lint / Lint Code (push) Failing after -30m11s
Build , Vet Test, and Lint / Build (push) Successful in -30m12s
Tests / Unit Tests (push) Successful in -30m49s
Tests / Integration Tests (push) Failing after -30m59s
2026-04-01 15:34:09 +02:00
362 changed files with 54769 additions and 7023 deletions
+11
View File
@@ -27,6 +27,17 @@ jobs:
with:
name: coverage-report
path: coverage.html
race-tests:
name: Race Detector
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v6
- name: Set up Go
uses: actions/setup-go@v6
with:
go-version: "1.24"
- name: Run unit tests with the race detector
run: go test -race -count=1 ./pkg/...
integration-tests:
name: Integration Tests
runs-on: ubuntu-latest
+2 -1
View File
@@ -28,4 +28,5 @@ test.db
/testserver
tests/data/
node_modules/
resolvespec-js/dist/
clients/resolvespec-js/dist/
.codex
+1
View File
@@ -30,6 +30,7 @@
"linters": {
"enable": [
"gocritic",
"gosec",
"misspell",
"revive"
],
+37 -9
View File
@@ -1,9 +1,21 @@
.PHONY: test test-unit test-integration docker-up docker-down clean
# Container compose command: podman if installed, else docker
COMPOSE ?= $(shell command -v podman >/dev/null 2>&1 && echo "podman compose" || echo "docker compose")
.PHONY: testserver-up testserver-down testserver-smoke test test-unit test-race test-integration docker-up docker-down clean
GOLANGCI_LINT := $(shell go env GOPATH)/bin/golangci-lint
# Run all unit tests
test-unit:
@echo "Running unit tests..."
@go test ./pkg/resolvespec ./pkg/restheadspec -v -cover
@go test ./pkg/... -v -cover
# Run all unit tests under the race detector (kept separate from coverage:
# race builds are 2-10x slower). Only races on executed paths are reported,
# so this covers every package rather than a subset.
test-race:
@echo "Running unit tests with the race detector..."
@go test -race -count=1 ./pkg/...
# Run all integration tests (requires PostgreSQL)
test-integration:
@@ -11,7 +23,7 @@ test-integration:
@go test -tags=integration ./pkg/resolvespec ./pkg/restheadspec -v
# Run all tests (unit + integration)
test: test-unit test-integration
test: test-unit test-race test-integration
release-version: ## Create and push a release with specific version (use: make release-version VERSION=v1.2.3 or make release-version to auto-increment)
@if [ -z "$(VERSION)" ]; then \
@@ -49,7 +61,9 @@ release-version: ## Create and push a release with specific version (use: make r
lint: ## Run linter
@echo "Running linter..."
@if command -v golangci-lint > /dev/null; then \
@if [ -x "$(GOLANGCI_LINT)" ]; then \
"$(GOLANGCI_LINT)" run --config=.golangci.json; \
elif command -v golangci-lint > /dev/null; then \
golangci-lint run --config=.golangci.json; \
else \
echo "golangci-lint not installed. Install with: go install github.com/golangci/golangci-lint/cmd/golangci-lint@latest"; \
@@ -58,7 +72,9 @@ lint: ## Run linter
lintfix: ## Run linter
@echo "Running linter..."
@if command -v golangci-lint > /dev/null; then \
@if [ -x "$(GOLANGCI_LINT)" ]; then \
"$(GOLANGCI_LINT)" run --config=.golangci.json --fix; \
elif command -v golangci-lint > /dev/null; then \
golangci-lint run --config=.golangci.json --fix; \
else \
echo "golangci-lint not installed. Install with: go install github.com/golangci/golangci-lint/cmd/golangci-lint@latest"; \
@@ -69,7 +85,7 @@ lintfix: ## Run linter
# Start PostgreSQL for integration tests
docker-up:
@echo "Starting PostgreSQL container..."
@podman compose up -d postgres-test
@$(COMPOSE) up -d postgres-test
@echo "Waiting for PostgreSQL to be ready..."
@sleep 5
@echo "PostgreSQL is ready!"
@@ -77,12 +93,23 @@ docker-up:
# Stop PostgreSQL container
docker-down:
@echo "Stopping PostgreSQL container..."
@podman compose down
@$(COMPOSE) down
# Test server + PostgreSQL in containers (dbtrace enabled)
testserver-up:
@$(COMPOSE) up -d --build postgres-test testserver
testserver-down:
@$(COMPOSE) down
testserver-smoke:
@COMPOSE="$(COMPOSE)" scripts/testserver-smoke.sh
# Clean up Docker volumes and test data
clean:
@echo "Cleaning up..."
@podman compose down -v
@$(COMPOSE) down -v
@echo "Cleanup complete!"
# Run integration tests with Docker (full workflow)
@@ -107,7 +134,8 @@ coverage-integration:
help:
@echo "Available targets:"
@echo " test-unit - Run unit tests"
@echo " test-unit - Run unit tests for all packages (./pkg/...)"
@echo " test-race - Run unit tests for all packages with -race"
@echo " test-integration - Run integration tests (requires PostgreSQL)"
@echo " test - Run all tests"
@echo " docker-up - Start PostgreSQL container"
+326 -146
View File
@@ -13,70 +13,69 @@ ResolveSpec is a flexible and powerful REST API specification and implementation
All share the same core architecture and provide dynamic data querying, relationship preloading, and complex filtering.
![1.00](./generated_slogan.webp)
## Table of Contents
* [Features](#features)
* [Installation](#installation)
* [Quick Start](#quick-start)
* [ResolveSpec (Body-Based API)](#resolvespec---body-based-api)
* [RestHeadSpec (Header-Based API)](#restheadspec---header-based-api)
* [ResolveMCP (MCP Server)](#resolvemcp---mcp-server)
* [Architecture](#architecture)
* [API Structure](#api-structure)
* [RestHeadSpec Overview](#restheadspec-header-based-api)
* [Example Usage](#example-usage)
* [Testing](#testing)
* [Additional Packages](#additional-packages)
* [Security Considerations](#security-considerations)
* [What's New](#whats-new)
- [Features](#features)
- [Installation](#installation)
- [Quick Start](#quick-start)
- [ResolveSpec (Body-Based API)](#resolvespec---body-based-api)
- [RestHeadSpec (Header-Based API)](#restheadspec---header-based-api)
- [ResolveMCP (MCP Server)](#resolvemcp---mcp-server)
- [Architecture](#architecture)
- [API Structure](#api-structure)
- [RestHeadSpec Overview](#restheadspec-header-based-api)
- [Example Usage](#example-usage)
- [Testing](#testing)
- [Additional Packages](#additional-packages)
- [Security Considerations](#security-considerations)
- [What's New](#whats-new)
## Features
### Core Features
* **Dynamic Data Querying**: Select specific columns and relationships to return
* **Relationship Preloading**: Load related entities with custom column selection and filters
* **Complex Filtering**: Apply multiple filters with various operators
* **Sorting**: Multi-column sort support
* **Pagination**: Built-in limit/offset and cursor-based pagination (both ResolveSpec and RestHeadSpec)
* **Computed Columns**: Define virtual columns for complex calculations
* **Custom Operators**: Add custom SQL conditions when needed
* **🆕 Recursive CRUD Handler**: Automatically handle nested object graphs with foreign key resolution and per-record operation control via `_request` field
- **Dynamic Data Querying**: Select specific columns and relationships to return
- **Relationship Preloading**: Load related entities with custom column selection and filters
- **Complex Filtering**: Apply multiple filters with various operators
- **Sorting**: Multi-column sort support
- **Pagination**: Built-in limit/offset and cursor-based pagination (both ResolveSpec and RestHeadSpec)
- **Computed Columns**: Define virtual columns for complex calculations
- **Custom Operators**: Add custom SQL conditions when needed
- **🆕 One Transaction Per Request**: Every statement and DB-touching hook of a request runs on one transaction; `OnTxBegin` hook stamps transaction-local settings (RLS) first. See [pkg/common/TRANSACTIONS.md](pkg/common/TRANSACTIONS.md)
- **🆕 Recursive CRUD Handler**: Automatically handle nested object graphs with foreign key resolution and per-record operation control via `_request` field
### Architecture (v2.0+)
* **🆕 Database Agnostic**: Works with GORM, Bun, or any database layer through adapters
* **🆕 Router Flexible**: Integrates with Gorilla Mux, Gin, Echo, or custom routers
* **🆕 Backward Compatible**: Existing code works without changes
* **🆕 Better Testing**: Mockable interfaces for easy unit testing
- **🆕 Database Agnostic**: Works with GORM, Bun, or any database layer through adapters
- **🆕 Router Flexible**: Integrates with Gorilla Mux, Gin, Echo, or custom routers
- **🆕 Backward Compatible**: Existing code works without changes
- **🆕 Better Testing**: Mockable interfaces for easy unit testing
### ResolveMCP (v3.2+)
* **🆕 MCP Server**: Expose any registered database model as Model Context Protocol tools and resources
* **🆕 AI-Ready Descriptions**: Tool descriptions include the full column schema, primary key, nullable flags, and relations — giving AI models everything they need to query correctly without guessing
* **🆕 Four Tools Per Model**: `read_`, `create_`, `update_`, `delete_` tools auto-registered per model
* **🆕 Full Query Support**: Filters, sort, limit/offset, cursor pagination, column selection, and relation preloading all available as tool parameters
* **🆕 HTTP/SSE Transport**: Standards-compliant SSE transport for use with Claude Desktop, Cursor, and any MCP-compatible client
* **🆕 Lifecycle Hooks**: Same Before/After hook system as ResolveSpec for auth and side-effects
- **🆕 MCP Server**: Expose any registered database model as Model Context Protocol tools and resources
- **🆕 AI-Ready Descriptions**: Tool descriptions include the full column schema, primary key, nullable flags, and relations — giving AI models everything they need to query correctly without guessing
- **🆕 Four Tools Per Model**: `read_`, `create_`, `update_`, `delete_` tools auto-registered per model
- **🆕 Full Query Support**: Filters, sort, limit/offset, cursor pagination, column selection, and relation preloading all available as tool parameters
- **🆕 HTTP/SSE Transport**: Standards-compliant SSE transport for use with Claude Desktop, Cursor, and any MCP-compatible client
- **🆕 Lifecycle Hooks**: Same Before/After hook system as ResolveSpec for auth and side-effects
### RestHeadSpec (v2.1+)
* **🆕 Header-Based API**: All query options passed via HTTP headers instead of request body
* **🆕 Lifecycle Hooks**: Before/after hooks for create, read, update, and delete operations
* **🆕 Cursor Pagination**: Efficient cursor-based pagination with complex sort support
* **🆕 Multiple Response Formats**: Simple, detailed, and Syncfusion-compatible formats
* **🆕 Single Record as Object**: Automatically normalize single-element arrays to objects (enabled by default)
* **🆕 Advanced Filtering**: Field filters, search operators, AND/OR logic, and custom SQL
* **🆕 Base64 Encoding**: Support for base64-encoded header values
- **🆕 Header-Based API**: All query options passed via HTTP headers instead of request body
- **🆕 Lifecycle Hooks**: Before/after hooks for create, read, update, and delete operations
- **🆕 Cursor Pagination**: Efficient cursor-based pagination with complex sort support
- **🆕 Multiple Response Formats**: Simple, detailed, and Syncfusion-compatible formats
- **🆕 Single Record as Object**: Automatically normalize single-element arrays to objects (enabled by default)
- **🆕 Advanced Filtering**: Field filters, search operators, AND/OR logic, and custom SQL
- **🆕 Base64 Encoding**: Support for base64-encoded header values
### Routing & CORS (v3.0+)
* **🆕 Explicit Route Registration**: Routes created per registered model instead of dynamic lookups
* **🆕 OPTIONS Method Support**: Full OPTIONS method support returning model metadata
* **🆕 CORS Headers**: Comprehensive CORS support with all HeadSpec headers allowed
* **🆕 Better Route Control**: Customize routes per model with more flexibility
- **🆕 Explicit Route Registration**: Routes created per registered model instead of dynamic lookups
- **🆕 OPTIONS Method Support**: Full OPTIONS method support returning model metadata
- **🆕 CORS Headers**: Comprehensive CORS support with all HeadSpec headers allowed
- **🆕 Better Route Control**: Customize routes per model with more flexibility
## API Structure
@@ -130,11 +129,81 @@ X-DetailApi: true
For complete documentation including setup, headers, lifecycle hooks, cursor pagination, and more, see [pkg/restheadspec/README.md](pkg/restheadspec/README.md).
## Example Usage
For detailed examples of reading data, cursor pagination, recursive CRUD operations, filtering, sorting, and more, see [pkg/resolvespec/README.md](pkg/resolvespec/README.md).
## PostGIS & Vector (PostgreSQL only)
First-class support for PostGIS geometry/geography and pgvector columns in `resolvespec` + `restheadspec`. No extra dependencies. On non-Postgres databases the spatial/vector operators simply don't match.
### Column types (`pkg/spectypes`)
| Go type | SQL type | Wire / JSON |
| ----------------- | -------------- | ------------------------------------------------------- |
| `SqlGeometry` | `geometry` | JSON in/out = **GeoJSON**; also accepts EWKT / hex-EWKB |
| `SqlGeography` | `geography` | same as `SqlGeometry` |
| `SqlVector` | `vector` | `[]float32` ⇄ `[1,2,3]` |
| `SqlHalfVector` | `halfvec` | `[]float32` ⇄ `[1,2,3]` |
| `SqlSparseVector` | `sparsevec` | `{"dim":8,"indices":[1,4],"values":[0.5,0.2]}` |
| `SqlBitVector` | `bit`/`varbit` | bool array or `"1011"` string |
- Geometry `Value()` emits `SRID=<n>;<WKT>` (PostGIS implicit text→geometry cast; no wrapper function needed).
- Declare dimensioned types with a tag: `gorm:"type:vector(1536)"` — the tag wins over the canonical name in metadata/OpenAPI.
- Metadata endpoint and OpenAPI schema report `geometry`/`vector`/`halfvec`/`sparsevec`/`bit`.
### Spatial filter operators
`value` is a geometry (GeoJSON object, EWKT string, or hex-EWKB) unless noted.
| Operator | Value shape |
| ----------------------------------------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------- |
| `st_intersects`, `st_contains`, `st_within`, `st_covers`, `st_coveredby`, `st_overlaps`, `st_touches`, `st_crosses`, `st_equals`, `st_disjoint` | geometry |
| `st_dwithin` | `{"geom": <geometry>, "distance": <meters>}` |
| `bbox` (alias `&&`) | geometry, or `{"bbox":[minx,miny,maxx,maxy],"srid":4326}` |
### Vector similarity filter operators
| Operator | pgvector op | Value shape |
| -------------------------------- | ----------- | ----------------------------------------------------------------- |
| `l2_within` / `euclidean_within` | `<->` | `{"vector":[...], "distance": <n>}` |
| `cosine_within` | `<=>` | same (also `"lt"`/`"lte"`/`"gt"`/`"gte"` instead of `"distance"`) |
| `ip_within` / `inner_within` | `<#>` | same |
### KNN search (ordering + distance column)
**resolvespec** — `options.vector_search`:
```json
{
"options": {
"vector_search": {
"column": "embedding",
"vector": [0.1, 0.2, 0.3],
"metric": "cosine",
"as": "_distance",
"direction": "asc"
}
}
}
```
Orders rows by distance; when `as` is set, returns the distance as an extra column (all model columns are auto-selected).
`metric`: `l2` (default) | `cosine` | `ip`.
**restheadspec** — headers:
```HTTP
X-Vector-Search-embedding: cosine
X-Vector-Search-Vector: [0.1,0.2,0.3]
X-Vector-Search-As: _distance
X-Vector-Search-Dir: asc
```
Spatial/vector filters via headers: `X-SpatialFilter-<col>` / `X-VectorFilter-<col>` with a JSON operator object, e.g.
`X-SpatialFilter-geom: {"op":"st_dwithin","geom":"SRID=4326;POINT(0 0)","distance":1000}`
(optional `"logic":"or"`).
## Installation
```Shell
@@ -275,25 +344,25 @@ Your Application Code
### Supported Database Layers
* **GORM** - Full support for PostgreSQL, SQLite, MSSQL
* **Bun** - Full support for PostgreSQL, SQLite, MSSQL
* **Native SQL** - Standard library `*sql.DB` with all supported databases
* **Custom ORMs** - Implement the `Database` interface
- **GORM** - Full support for PostgreSQL, SQLite, MSSQL
- **Bun** - Full support for PostgreSQL, SQLite, MSSQL
- **Native SQL** - Standard library `*sql.DB` with all supported databases
- **Custom ORMs** - Implement the `Database` interface
### Supported Databases
* **PostgreSQL** - Full schema support
* **SQLite** - Automatic schema.table to schema_table translation
* **Microsoft SQL Server** - Full schema support
* **MongoDB** - NoSQL document database (via MQTTSpec and custom handlers)
- **PostgreSQL** - Full schema support
- **SQLite** - Automatic schema.table to schema_table translation
- **Microsoft SQL Server** - Full schema support
- **MongoDB** - NoSQL document database (via MQTTSpec and custom handlers)
### Supported Routers
* **Gorilla Mux** (built-in support with `SetupRoutes()`)
* **BunRouter** (built-in support with `SetupBunRouterWithResolveSpec()`)
* **Gin** (manual integration, see examples above)
* **Echo** (manual integration, see examples above)
* **Custom Routers** (implement request/response adapters)
- **Gorilla Mux** (built-in support with `SetupRoutes()`)
- **BunRouter** (built-in support with `SetupBunRouterWithResolveSpec()`)
- **Gin** (manual integration, see examples above)
- **Echo** (manual integration, see examples above)
- **Custom Routers** (implement request/response adapters)
## Testing
@@ -303,6 +372,14 @@ ResolveSpec is designed for testability with mockable interfaces. For testing ex
- [RestHeadSpec Testing](pkg/restheadspec/README.md#testing)
- [WebSocketSpec Testing](pkg/websocketspec/README.md)
### Test Server (dbtrace, real PostgreSQL)
- `make testserver-up` / `make testserver-down`: testserver + PostgreSQL via compose (host networking)
- `make testserver-smoke`: create, read, update, delete, batch create/delete against the testserver
- Ports: testserver `8123`, PostgreSQL `8124`
- `dbtrace` logs per request `tx`, `tx_queries`, `pooled`, `raw`; `pooled=0` is the target
- Integration tests default to PostgreSQL on `localhost:8124`
## Continuous Integration
ResolveSpec uses GitHub Actions for automated testing and quality checks. The CI pipeline runs on every push and pull request.
@@ -311,10 +388,10 @@ ResolveSpec uses GitHub Actions for automated testing and quality checks. The CI
The project includes automated workflows that:
* **Test**: Run all tests with race detection and code coverage
* **Lint**: Check code quality with golangci-lint
* **Build**: Verify the project builds successfully
* **Multi-version**: Test against multiple Go versions (1.23.x, 1.24.x)
- **Test**: Run all tests with race detection and code coverage
- **Lint**: Check code quality with golangci-lint
- **Build**: Verify the project builds successfully
- **Multi-version**: Test against multiple Go versions (1.23.x, 1.24.x)
### Running Tests Locally
@@ -336,9 +413,9 @@ golangci-lint run
The project includes comprehensive test coverage:
* **Unit Tests**: Individual component testing
* **Integration Tests**: End-to-end API testing
* **CRUD Tests**: Standalone tests for both ResolveSpec and RestHeadSpec APIs
- **Unit Tests**: Individual component testing
- **Integration Tests**: End-to-end API testing
- **CRUD Tests**: Standalone tests for both ResolveSpec and RestHeadSpec APIs
To run only the CRUD standalone tests:
@@ -369,6 +446,7 @@ ResolveSpec includes several complementary packages that work together to provid
The core body-based REST API with GraphQL-like capabilities.
**Key Features**:
- JSON request body with operation and options
- Recursive CRUD with nested object support
- Cursor and offset pagination
@@ -382,6 +460,7 @@ For complete documentation, see [pkg/resolvespec/README.md](pkg/resolvespec/READ
Alternative REST API where query options are passed via HTTP headers.
**Key Features**:
- All query options via HTTP headers
- Same capabilities as ResolveSpec
- Cleaner separation of data and metadata
@@ -394,6 +473,7 @@ For complete documentation, see [pkg/restheadspec/README.md](pkg/restheadspec/RE
Expose any registered model as Model Context Protocol tools and resources consumable by AI models over HTTP/SSE.
**Key Features**:
- Four tools per model: `read_`, `create_`, `update_`, `delete_`
- Rich AI-readable descriptions: column names, types, primary key, nullable flags, and preloadable relations
- Full query support: filters, sort, limit/offset, cursor pagination, column selection, preloads
@@ -407,6 +487,7 @@ For complete documentation, see [pkg/resolvemcp/](pkg/resolvemcp/).
Execute SQL functions and queries through a simple HTTP API with header-based parameters.
**Key Features**:
- Direct SQL function invocation
- Header-based parameter passing
- Automatic pagination and counting
@@ -415,16 +496,30 @@ Execute SQL functions and queries through a simple HTTP API with header-based pa
For complete documentation, see [pkg/funcspec/](pkg/funcspec/).
#### Clients
All clients are under [clients/](clients/README.md); wire behaviour is identical across them.
| Client | Language | Specs | Docs |
| -------------------- | -------------- | ---------------------------------------------------- | ---------------------------------------------- |
| `resolvespec-js` | TypeScript | ResolveSpec, HeaderSpec, FunctionSpec, WebSocketSpec | [README](clients/resolvespec-js/README.md) |
| `resolvespec-python` | Python >= 3.11 | ResolveSpec, HeaderSpec, FunctionSpec, WebSocketSpec | [README](clients/resolvespec-python/README.md) |
| `resolvespec-go` | Go | ResolveSpec, FunctionSpec | [README](clients/resolvespec-go/README.md) |
| `resolvespec-rs` | Rust | ResolveSpec, FunctionSpec | [README](clients/resolvespec-rs/README.md) |
| `resolvespec-cs` | C# (.NET 8) | ResolveSpec, FunctionSpec | [README](clients/resolvespec-cs/README.md) |
| `resolvespec-dart` | Dart / Flutter | ResolveSpec, FunctionSpec | [README](clients/resolvespec-dart/README.md) |
#### ResolveSpec JS - TypeScript Client Library
TypeScript/JavaScript client library supporting all three REST and WebSocket protocols.
**Clients**:
- Body-based REST client (`read`, `create`, `update`, `deleteEntity`)
- Header-based REST client (`HeaderSpecClient`)
- WebSocket client (`WebSocketClient`) with CRUD, subscriptions, heartbeat, reconnect
For complete documentation, see [resolvespec-js/README.md](resolvespec-js/README.md).
For complete documentation, see [clients/resolvespec-js/README.md](clients/resolvespec-js/README.md).
### Real-Time Communication
@@ -433,6 +528,7 @@ For complete documentation, see [resolvespec-js/README.md](resolvespec-js/README
Real-time bidirectional communication with full CRUD operations and subscriptions.
**Key Features**:
- Persistent WebSocket connections
- Real-time subscriptions to entity changes
- Automatic push notifications
@@ -446,6 +542,7 @@ For complete documentation, see [pkg/websocketspec/README.md](pkg/websocketspec/
MQTT-based database operations ideal for IoT and mobile applications.
**Key Features**:
- Embedded or external MQTT broker support
- QoS 1 (at-least-once delivery)
- Real-time subscriptions
@@ -461,6 +558,7 @@ For complete documentation, see [pkg/mqttspec/README.md](pkg/mqttspec/README.md)
Flexible, interface-driven static file server.
**Key Features**:
- Router-agnostic with standard `http.Handler`
- Multiple filesystem backends (local, zip, embedded)
- Pluggable cache, MIME, and fallback policies
@@ -468,6 +566,7 @@ Flexible, interface-driven static file server.
- 140+ MIME types including modern formats
**Quick Example**:
```go
import "github.com/bitechdev/ResolveSpec/pkg/server/staticweb"
@@ -492,6 +591,7 @@ For complete documentation, see [pkg/server/staticweb/README.md](pkg/server/stat
Comprehensive event handling system for real-time event publishing and cross-instance communication.
**Key Features**:
- Multiple event sources (database, websockets, frontend, system)
- Multiple providers (in-memory, Redis Streams, NATS, PostgreSQL)
- Pattern-based subscriptions
@@ -506,13 +606,35 @@ For complete documentation, see [pkg/eventbroker/README.md](pkg/eventbroker/READ
Centralized management of multiple database connections with support for PostgreSQL, SQLite, MSSQL, and MongoDB.
**Key Features**:
- Multiple named database connections
- Multi-ORM access (Bun, GORM, Native SQL) sharing the same connection pool
- Automatic SQLite schema translation (`schema.table` → `schema_table`)
- Health checks with auto-reconnect
- Background health checks (report status; they never close the pool)
- Prometheus metrics for monitoring
- Configuration-driven via YAML
- Per-connection statistics and management
- Per-connection statistics and management, including pool limit (`max`) and total connections ever opened (`dbmanager_connection_pool_size{state="max"}`, `dbmanager_connections_opened_total`)
**How to use it correctly**:
```go
mgr, err := dbmanager.NewManager(cfg) // or dbmanager.SetupManager(cfg) + GetInstance()
if err != nil { /* handle */ }
if err := mgr.Connect(ctx); err != nil { /* handle */ } // SetupManager does NOT connect
defer mgr.Close() // once, at shutdown
conn, _ := mgr.GetDefault()
db, _ := conn.Bun() // or conn.GORM() / conn.Native() / conn.Database()
handler := restheadspec.NewHandlerWithBun(db)
```
- **Fetch a handle once and keep it.** `Bun()`, `GORM()`, `Native()` and `Database()` return handles over one long-lived `*sql.DB`. You do not need to re-fetch them per request, and they stay valid for the life of the connection.
- **Never close a handle yourself.** Closing a `*bun.DB`, `*gorm.DB` or the `*sql.DB` closes the shared pool for everyone. Only `mgr.Close()` (at shutdown) should close it. After `Close`, the handles are dead.
- **Don't reconnect to recover from errors.** `database/sql` already discards bad connections and dials new ones. The manager does not close the pool on errors or failed health checks. `conn.Reconnect(ctx)` is for explicit operator use only (for example after rotating credentials): on PostgreSQL it retires pooled connections without closing the pool, so held handles keep working. Other databases close and reopen the pool, which invalidates handles you already hold.
- **Bring your own `*sql.DB`.** `dbmanager.NewConnectionFromDB(name, type, db)` wraps a pool you opened. The manager never closes it (`Close` only logs a warning); you own it and must close it.
- **Set deadlines on request contexts.** `query_timeout` is applied to PostgreSQL as `statement_timeout` (server side) and TCP timeouts detect dead sockets, but pass a context with a deadline to your queries so callers fail fast.
- **Pool tuning.** Keep `conn_max_idle_time` below the shortest idle timeout of any NAT, load balancer or pgbouncer between you and the database (typically 60-240s). SQLite `:memory:` is pinned to a single connection.
- **Health checks** run every `health_check_interval` (default 15s; a negative value disables them) and publish Prometheus metrics. `enable_auto_reconnect` is deprecated and ignored.
For documentation, see [pkg/dbmanager/README.md](pkg/dbmanager/README.md).
@@ -524,13 +646,21 @@ For documentation, see [pkg/cache/README.md](pkg/cache/README.md).
#### Security
Authentication and authorization framework with hooks integration.
Authentication and authorization framework with hooks integration. Database-backed providers use PostgreSQL stored procedures by default, with a portable Direct mode (plain Go/SQL) for SQLite, MySQL, or Postgres without the procedures installed.
For documentation, see [pkg/security/README.md](pkg/security/README.md).
For documentation, see [pkg/security/README.md](pkg/security/README.md) (see "Direct Mode" for the SQLite/portable-SQL path).
#### Middleware
HTTP middleware collection for common tasks (CORS, logging, metrics, etc.).
HTTP middleware collection for common tasks (CORS, logging, metrics, rate limiting, etc.).
**Client request queue** (`middleware.ClientQueue`): limits how many requests each client runs concurrently and queues the rest first-in-first-out, smoothing bursts such as a page load that fires ~15 requests at once. Clients are identified by `X-Client-Id`, then `Authorization`, then the built-in session, then IP, so no client changes are required. Exposes burst, wait and queue-depth Prometheus metrics. Add it through the middleware slot of `SetupMuxRoutes` / `SetupBunRouterRoutes`:
```go
q := middleware.NewClientQueue(middleware.ClientQueueConfig{MaxConcurrent: 10})
defer q.Close()
restheadspec.SetupMuxRoutes(router, handler, middleware.Chain(authMiddleware, q.Middleware))
```
For documentation, see [pkg/middleware/README.md](pkg/middleware/README.md).
@@ -564,14 +694,31 @@ Configuration management with support for multiple formats and environments.
For documentation, see [pkg/config/README.md](pkg/config/README.md).
#### DB Trace
Per-request DB call counting (`tx`, `tx_queries`, `pooled`, `raw`) and pool logging. Off by default.
For documentation, see [pkg/dbtrace/README.md](pkg/dbtrace/README.md).
### Core Libraries
| Package | Purpose |
| ----------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------- |
| [`pkg/common`](pkg/common/) | Shared interfaces (database, request/response adapters), validation, recursive CRUD, request transactions ([TRANSACTIONS.md](pkg/common/TRANSACTIONS.md)) |
| [`pkg/modelregistry`](pkg/modelregistry/) | Model registration by schema/entity and per-model access rules |
| [`pkg/reflection`](pkg/reflection/) | Model/struct reflection helpers (primary keys, columns, relations) |
| [`pkg/spectypes`](pkg/spectypes/) | SQL-aware types (nullable, JSONB, PostGIS, vector) |
| [`pkg/logger`](pkg/logger/) | Logging used by all packages |
| [`pkg/testmodels`](pkg/testmodels/) | Shared test models and data for tests and the testserver |
## Security Considerations
* Implement proper authentication and authorization
* Validate all input parameters
* Use prepared statements (handled by GORM/Bun/your ORM)
* Implement rate limiting
* Control access at schema/entity level
* **New**: Database abstraction layer provides additional security through interface boundaries
- Implement proper authentication and authorization
- Validate all input parameters
- Use prepared statements (handled by GORM/Bun/your ORM)
- Implement rate limiting (`middleware.RateLimiter`) and per-client request queueing (`middleware.ClientQueue`)
- Control access at schema/entity level
- **New**: Database abstraction layer provides additional security through interface boundaries
## Contributing
@@ -587,122 +734,155 @@ This project is licensed under the MIT License - see the [LICENSE](LICENSE) file
## What's New
### Unreleased
**Single transaction per request**:
- **One tx per request**: hooks get the transaction in `hookCtx.Tx`, never the pool (`BeforeHandle` runs before any tx and must not touch the DB)
- **`OnTxBegin` hook**: all specs (mqttspec re-exports websocketspec's); fires once, first, in every tx; error or abort rolls back with no detail to the client
- **Second short tx**: create/update re-fetch, `BeforeScan` and post-commit hooks (`AfterCreate`, `AfterUpdate`, restheadspec `AfterRead`, funcspec `BeforeResponse`) run on a new tx after the first commits
- **Delete**: single and batch delete, hooks included, in one tx
- **websocketspec / mqttspec**: one tx per message; begin/commit failures answer `transaction_error`
- **resolvemcp**: read, create, update, delete transactional
- **RLS stamping**: `SecurityList.SetTxSettings(fn)`; `RegisterSecurityHooks` of every spec stamps `set_config(name, value, true)` on `OnTxBegin`; fails closed
- **New**: `common.RunRequestTx`, `common.TxContext`, `common.TxHookName`
- **Behavior changes**: `AfterDelete` failure now rolls the delete back; funcspec begin/commit failure answers 500 `transaction_error`
**Clients**: Go, Rust, C# and Dart clients for ResolveSpec and FunctionSpec under `clients/`.
**Test server**: compose uses host networking; ports `8123` (testserver) and `8124` (PostgreSQL), previously `8080` and `5434`.
### v3.2 (Latest - March 2026)
**ResolveMCP - Model Context Protocol Server (🆕)**:
* **MCP Tools**: Four tools auto-registered per model (`read_`, `create_`, `update_`, `delete_`) over HTTP/SSE transport
* **AI-Ready Descriptions**: Full column schema, primary key, nullable flags, and relation names surfaced in tool descriptions so AI models can query without guessing
* **Full Query Support**: Filters, sort, limit/offset, cursor pagination, column selection, and relation preloading all available as tool parameters
* **HTTP/SSE Transport**: Standards-compliant transport compatible with Claude Desktop, Cursor, and any MCP 2024-11-05 client
* **Lifecycle Hooks**: Same Before/After hook system as ResolveSpec for auth, auditing, and side-effects
* **MCP Resources**: Each model also exposed as a named resource for direct data access by AI clients
- **MCP Tools**: Four tools auto-registered per model (`read_`, `create_`, `update_`, `delete_`) over HTTP/SSE transport
- **AI-Ready Descriptions**: Full column schema, primary key, nullable flags, and relation names surfaced in tool descriptions so AI models can query without guessing
- **Full Query Support**: Filters, sort, limit/offset, cursor pagination, column selection, and relation preloading all available as tool parameters
- **HTTP/SSE Transport**: Standards-compliant transport compatible with Claude Desktop, Cursor, and any MCP 2024-11-05 client
- **Lifecycle Hooks**: Same Before/After hook system as ResolveSpec for auth, auditing, and side-effects
- **MCP Resources**: Each model also exposed as a named resource for direct data access by AI clients
### v3.1 (February 2026)
**SQLite Schema Translation (🆕)**:
* **Automatic Schema Translation**: SQLite support with automatic `schema.table` to `schema_table` conversion
* **Database Agnostic Models**: Write models once, use across PostgreSQL, SQLite, and MSSQL
* **Transparent Handling**: Translation occurs automatically in all operations (SELECT, INSERT, UPDATE, DELETE, preloads)
* **All ORMs Supported**: Works with Bun, GORM, and Native SQL adapters
- **Automatic Schema Translation**: SQLite support with automatic `schema.table` to `schema_table` conversion
- **Database Agnostic Models**: Write models once, use across PostgreSQL, SQLite, and MSSQL
- **Transparent Handling**: Translation occurs automatically in all operations (SELECT, INSERT, UPDATE, DELETE, preloads)
- **All ORMs Supported**: Works with Bun, GORM, and Native SQL adapters
### v3.0 (December 2025)
**Explicit Route Registration (🆕)**:
* **Breaking Change**: Routes are now created explicitly for each registered model
* **Better Control**: Customize routes per model with more flexibility
* **Registration Order**: Models must be registered BEFORE calling SetupMuxRoutes/SetupBunRouterRoutes
* **Benefits**: More flexible routing, easier to add custom routes per model, better performance
- **Breaking Change**: Routes are now created explicitly for each registered model
- **Better Control**: Customize routes per model with more flexibility
- **Registration Order**: Models must be registered BEFORE calling SetupMuxRoutes/SetupBunRouterRoutes
- **Benefits**: More flexible routing, easier to add custom routes per model, better performance
**OPTIONS Method & CORS Support (🆕)**:
* **OPTIONS Endpoint**: Full OPTIONS method support for CORS preflight requests
* **Metadata Response**: OPTIONS returns model metadata (same as GET /metadata)
* **CORS Headers**: Comprehensive CORS headers on all responses
* **Header Support**: All HeadSpec custom headers (`X-Select-Fields`, `X-FieldFilter-*`, etc.) allowed
* **No Auth on OPTIONS**: CORS preflight requests don't require authentication
* **Configurable**: Customize CORS settings via `common.CORSConfig`
- **OPTIONS Endpoint**: Full OPTIONS method support for CORS preflight requests
- **Metadata Response**: OPTIONS returns model metadata (same as GET /metadata)
- **CORS Headers**: Comprehensive CORS headers on all responses
- **Header Support**: All HeadSpec custom headers (`X-Select-Fields`, `X-FieldFilter-*`, etc.) allowed
- **No Auth on OPTIONS**: CORS preflight requests don't require authentication
- **Configurable**: Customize CORS settings via `common.CORSConfig`
### v2.1
**Cursor Pagination for ResolveSpec (🆕 Dec 9, 2025)**:
* **Cursor-Based Pagination**: Efficient cursor pagination now available in ResolveSpec (body-based API)
* **Consistent with RestHeadSpec**: Both APIs now support cursor pagination for feature parity
* **Multi-Column Sort Support**: Works seamlessly with complex sorting requirements
* **Better Performance**: Improved performance for large datasets compared to offset pagination
* **SQL Safety**: Proper SQL sanitization for cursor values
- **Cursor-Based Pagination**: Efficient cursor pagination now available in ResolveSpec (body-based API)
- **Consistent with RestHeadSpec**: Both APIs now support cursor pagination for feature parity
- **Multi-Column Sort Support**: Works seamlessly with complex sorting requirements
- **Better Performance**: Improved performance for large datasets compared to offset pagination
- **SQL Safety**: Proper SQL sanitization for cursor values
**Recursive CRUD Handler (🆕 Nov 11, 2025)**:
* **Nested Object Graphs**: Automatically handle complex object hierarchies with parent-child relationships
* **Foreign Key Resolution**: Automatic propagation of parent IDs to child records
* **Per-Record Operations**: Control create/update/delete operations per record via `_request` field
* **Transaction Safety**: All nested operations execute atomically within database transactions
* **Relationship Detection**: Automatic detection of belongsTo, hasMany, hasOne, and many2many relationships
* **Deep Nesting Support**: Handle relationships at any depth level
* **Mixed Operations**: Combine insert, update, and delete operations in a single request
- **Nested Object Graphs**: Automatically handle complex object hierarchies with parent-child relationships
- **Foreign Key Resolution**: Automatic propagation of parent IDs to child records
- **Per-Record Operations**: Control create/update/delete operations per record via `_request` field
- **Transaction Safety**: All nested operations execute atomically within database transactions
- **Relationship Detection**: Automatic detection of belongsTo, hasMany, hasOne, and many2many relationships
- **Deep Nesting Support**: Handle relationships at any depth level
- **Mixed Operations**: Combine insert, update, and delete operations in a single request
**Primary Key Improvements (Nov 11, 2025)**:
* **GetPrimaryKeyName**: Enhanced primary key detection for better preload and ID field handling
* **Better GORM/Bun Support**: Improved compatibility with both ORMs for primary key operations
* **Computed Column Support**: Fixed computed columns functionality across handlers
- **GetPrimaryKeyName**: Enhanced primary key detection for better preload and ID field handling
- **Better GORM/Bun Support**: Improved compatibility with both ORMs for primary key operations
- **Computed Column Support**: Fixed computed columns functionality across handlers
**Database Adapter Enhancements (Nov 11, 2025)**:
* **Bun ORM Relations**: Using Scan model method for better has-many and many-to-many relationship handling
* **Model Method Support**: Enhanced query building with proper model registration
* **Improved Type Safety**: Better handling of relationship queries with type-aware scanning
- **Bun ORM Relations**: Using Scan model method for better has-many and many-to-many relationship handling
- **Model Method Support**: Enhanced query building with proper model registration
- **Improved Type Safety**: Better handling of relationship queries with type-aware scanning
**RestHeadSpec - Header-Based REST API**:
* **Header-Based Querying**: All query options via HTTP headers instead of request body
* **Lifecycle Hooks**: Before/after hooks for create, read, update, delete operations
* **Cursor Pagination**: Efficient cursor-based pagination with complex sorting
* **Advanced Filtering**: Field filters, search operators, AND/OR logic
* **Multiple Response Formats**: Simple, detailed, and Syncfusion-compatible responses
* **Single Record as Object**: Automatically return single-element arrays as objects (default, toggleable via header)
* **Base64 Support**: Base64-encoded header values for complex queries
* **Type-Aware Filtering**: Automatic type detection and conversion for filters
- **Header-Based Querying**: All query options via HTTP headers instead of request body
- **Lifecycle Hooks**: Before/after hooks for create, read, update, delete operations
- **Cursor Pagination**: Efficient cursor-based pagination with complex sorting
- **Advanced Filtering**: Field filters, search operators, AND/OR logic
- **Multiple Response Formats**: Simple, detailed, and Syncfusion-compatible responses
- **Single Record as Object**: Automatically return single-element arrays as objects (default, toggleable via header)
- **Base64 Support**: Base64-encoded header values for complex queries
- **Type-Aware Filtering**: Automatic type detection and conversion for filters
**Core Improvements**:
* Better model registry with schema.table format support
* Enhanced validation and error handling
* Improved reflection safety
* Fixed COUNT query issues with table aliasing
* Better pointer handling throughout the codebase
* **Comprehensive Test Coverage**: Added standalone CRUD tests for both ResolveSpec and RestHeadSpec
- Better model registry with schema.table format support
- Enhanced validation and error handling
- Improved reflection safety
- Fixed COUNT query issues with table aliasing
- Better pointer handling throughout the codebase
- **Comprehensive Test Coverage**: Added standalone CRUD tests for both ResolveSpec and RestHeadSpec
### v2.0
**Breaking Changes**:
* **None!** Full backward compatibility maintained
- **None!** Full backward compatibility maintained
**New Features**:
* **Database Abstraction**: Support for GORM, Bun, and custom ORMs
* **Router Flexibility**: Works with any HTTP router through adapters
* **BunRouter Integration**: Built-in support for uptrace/bunrouter
* **Better Architecture**: Clean separation of concerns with interfaces
* **Enhanced Testing**: Mockable interfaces for comprehensive testing
- **Database Abstraction**: Support for GORM, Bun, and custom ORMs
- **Router Flexibility**: Works with any HTTP router through adapters
- **BunRouter Integration**: Built-in support for uptrace/bunrouter
- **Better Architecture**: Clean separation of concerns with interfaces
- **Enhanced Testing**: Mockable interfaces for comprehensive testing
**Performance Improvements**:
* More efficient query building through interface design
* Reduced coupling between components
* Better memory management with interface boundaries
- More efficient query building through interface design
- Reduced coupling between components
- Better memory management with interface boundaries
# Security Policy
## Reporting a vulnerability
Please do not open a public issue for security problems.
Report privately through GitHub: Security → Report a vulnerability
(https://github.com/bitechdev/ResolveSpec/security/advisories/new),
or email hein@bitechsystems.co.za / hein@warky.dev
You'll get an acknowledgement within 7 days. We aim to release a fix within
90 days and will credit reporters in the advisory unless they prefer otherwise.
## Acknowledgments
* Inspired by REST, OData, and GraphQL's flexibility
* **Header-based approach**: Inspired by REST best practices and clean API design
* **Database Support**: [GORM](https://gorm.io) and [Bun](https://bun.uptrace.dev/)
* **Router Support**: Gorilla Mux (built-in), BunRouter, Gin, Echo, and others through adapters
* Slogan generated using DALL-E
* AI used for documentation checking and correction
* Community feedback and contributions that made v2.0 and v2.1 possible
- Inspired by REST, OData, and GraphQL's flexibility
- **Header-based approach**: Inspired by REST best practices and clean API design
- **Database Support**: [GORM](https://gorm.io) and [Bun](https://bun.uptrace.dev/)
- **Router Support**: Gorilla Mux (built-in), BunRouter, Gin, Echo, and others through adapters
- Slogan generated using DALL-E
- AI used for documentation checking and correction
- Community feedback and contributions that made v2.0 and v2.1 possible
![1.00](./generated_slogan.webp)
+145
View File
@@ -0,0 +1,145 @@
# resolvemcp rewrite plan
Source: `audit/pkg/resolvemcp.audit.md`. Status: plan only, no code changed.
## Goal
Replace 4 tools + 1 resource per model with a fixed set of meta tools.
Endpoint guarded by OAuth / session token / API key; tools run as the authenticated caller.
Same rules as resolvespec CRUD, plus guardrails.
## Decisions
| Topic | Decision |
|---|---|
| Tools | Fixed meta tools; per-model tools/resources removed (breaking) |
| Functions | Explicit registry `Handler.RegisterFunction`; two kinds: Go callback (`func(ctx, tx, args)` + JSON-schema params) and SQL procedure by name (declared params); both behind `call_function`, run in tx with hooks |
| Create | `insert_into_table` included |
| Writes | update/delete by id **or** filters |
| Guardrails | require id or filters, max rows, `dry_run`, confirm token |
| Token scope | filter writes only; id writes = single row, no token |
| Confirm token store | in-memory, TTL, bound to user/table/filter hash; lost on restart, single instance |
| Read limits | server caps: limit, offset, batch, preload depth, timeout |
| Model exposure | all registered models visible; rules only restrict operations |
| Visibility | list tools show only what the caller may do (rules) |
| Identity | the authenticated caller's `UserContext`; **no fixed MCP user, no `SetUsername`, no service session, no background refresh** |
| Guard | endpoint always requires one of: OAuth bearer, session token, API key; no guest/optional mode |
| API key login | new `DatabaseAuthenticator.LoginWithAPIKey(ctx, rawKey)` + procedure `resolvespec_login_api_key`; validates key via keystore, creates session, returns `LoginResponse` |
| Session SQL | procedure mode + direct-SQL fallback (`ShouldUseProcedure`), same as `Login` |
| OAuth routes | `oauth2.go`/`oauth2_server.go` kept, part of the guard |
| Annotations | opt-in `Config.EnableAnnotations`, via `BeforeHandle` |
## Open
- None.
## Tools
| Tool | Purpose |
|---|---|
| `list_tables` | visible `schema.entity` + allowed ops |
| `describe_table` | columns, PK, relations, writable columns, rules, limits |
| `select_table` | filters, sort, columns, preloads, cursor; capped |
| `insert_into_table` | one or batch (capped); column allowlist |
| `update_table` | validated keys; id or filters; guardrails |
| `delete_from_table` | id or filters; guardrails |
| `list_functions` | registered functions + parameter schemas |
| `call_function` | validated args; tx + hooks + rules |
## Config additions
| Field | Purpose |
|---|---|
| `DefaultLimit`, `MaxLimit`, `MaxOffset` | read paging caps |
| `MaxBatch` | insert batch cap |
| `MaxPreloadDepth` | preload cap |
| `MaxWriteRows` | filter-write row cap |
| `QueryTimeout` | per-call context timeout |
| `ConfirmTTL` | confirm token lifetime |
| `EnableAnnotations` | opt-in annotate tool |
## Guardrail rules
| Rule | Behaviour |
|---|---|
| Target required | update/delete with neither id nor filters rejected |
| Max rows | count matches inside tx; abort above `MaxWriteRows` |
| `dry_run` | returns match count + preview, no write |
| Confirm token | filter write: first call returns token + preview; second call with token executes; bound to user, table, filter hash; expires at `ConfirmTTL` |
| Id write | single row, no token |
## Work items
### 1. API key login (`pkg/security`)
- Existing: keystore has `ValidateKey` and `KeyStoreAuthenticator`; `Login` needs a password; no key-to-session path.
- Add `resolvespec_login_api_key` to `SQLNames` (default + override) and a SQL script beside the existing procedures. Contract: `p_success, p_error, p_data`, input raw key; hashes, validates active/non-expired key, creates session for the key's user.
- Add `DatabaseAuthenticator.LoginWithAPIKey(ctx, rawKey)`; procedure first, direct-SQL fallback via `ShouldUseProcedure`.
- Hashed lookup; same generic error for unknown, expired or inactive key; no key material in logs.
- Expose through the chain/composite authenticators so the middleware can accept it.
### 2. Endpoint guard
- Wire `security.NewAuthMiddleware` with a chain of OAuth bearer, session token (header/cookie) and API key.
- `SetupMux*`/`SetupBunRouter*` helpers require the guard; unauthenticated serving only when explicitly constructed without it, logged loudly. Remove `OptionalAuth*` from the MCP path.
- Caller `UserContext` flows to every tool call context; rules, RLS and `OnTxBegin` apply to that user.
### 3. Security fixes (audit #1-6)
- Put model rules in request context (`withRequestData`) and/or `AddRegistry` on construction.
- Add `BeforeCreate` -> `CheckModelCreateAllowed` (new in `pkg/security`).
- Call `BeforeHandle` first in `executeUpdate`.
- Validate create/update keys against `ColumnValidator`; reject unknown; resolve column names from model, not json tags.
- Apply row security to update/delete pre-read; fail if row not visible.
- Annotate tool: opt-in + `BeforeHandle`.
### 4. Limits (audit #7, #13)
- Apply default/max limit, max offset, batch cap, preload depth cap, timeout.
- Validate preload names against model relations.
- Count only when requested.
### 5. Error and panic surface (audit #9, #17)
- Stable error codes + short message to client.
- Details and stack logged server-side.
- Recover hook panics.
### 6. Update/create semantics (audit #10-12)
- `SET` from validated incoming keys only; explicit null supported.
- Lock row (`FOR UPDATE`) on update.
- Refetch and `After*` hooks inside the same tx, or report committed write with warning if not possible (see `audit/single_tran.md`).
### 7. Smaller fixes (audit #8, #14, #15)
- SSE pool: require `BaseURL` or cap/evict; allowlist Host.
- Uniform not-found vs hook error text.
- Add mutex to `HookRegistry`.
### 8. Meta tools
- New file for meta tools; reuse parse helpers and `buildModelInfo` for `describe_table`.
- Remove per-model register functions and resources.
- `RegisterModel` only registers to registry.
- Function registry (Go callback kind + SQL procedure kind) + validation of args against declared schema.
### 9. Tests
- Update `tx_test.go` (calls `executeRead/Create/Update/Delete`) and `tools_test.go`.
- New: rule enforcement, unknown keys, limits, guardrails (cap, dry_run, token expiry/binding), guard rejects unauthenticated, API key login (valid, expired, inactive, unknown), visibility filtering, `-race`.
- Check for existing test data first; ask before generating any.
### 10. Docs
- Rewrite `pkg/resolvemcp/README.md` cheatsheet style.
- Document `resolvespec_login_api_key` in `pkg/security` docs.
- Update root README references.
- Update audit file when findings are closed.
## Order
1. `LoginWithAPIKey` + procedure in `pkg/security` (1)
2. Guard + security fixes (2-3)
3. Limits, errors, update/create semantics (4-6)
4. Meta tools + function registry (8)
5. Smaller fixes (7)
6. Tests (9), docs (10)
## Breaking changes
- Per-model tools and resources gone.
- MCP endpoint requires authentication.
- Annotate tool off by default.
- Update/create reject unknown keys.
- Reads capped by default.
+677
View File
@@ -0,0 +1,677 @@
# Audit: cross-cutting findings across `pkg/*`
| | |
|---|---|
| **Scope** | all 23 packages under `pkg/` (64 065 non-test lines) |
| **Audit date** | 2026-09-29 |
| **Axes** | thread locking/waiting, slowness, security, panic handling & logging |
| **Threat model** | hostile internet client; request bodies, headers, query params, schema/table/column names all attacker-controlled |
This file records findings that are **not specific to one package** — they are
properties of the repository or patterns repeated across many packages. The
per-package audits reference this file rather than restating them.
## Findings
| # | Severity | Axis | Finding |
|---|---|---|---|
| X1 | **High** | locking | `-race` is never run anywhere; no package is ever race-checked |
| X2 | **High** | testing | `go test` runs against 2 of 23 packages; the other 21 are only compiled and vetted |
| X3 | **High** | testing | Every integration-test step is `continue-on-error: true` — integration failures cannot fail CI |
| X10 | **High** | security | Whole subsystems are declared, configured, documented and tested but never installed — including every protective middleware and the metrics provider |
| X4 | **Medium** | security | `gosec` is not enabled in `.golangci.json`; no SAST runs on a package set full of dynamic SQL |
| X5 | **Medium** | locking | Unsynchronized package-level mutable globals are the dominant concurrency pattern |
| X6 | **Medium** | security | Insecure-by-default transport across the board: `sslmode: disable`, `WithInsecure()`, no TLS in cache configs |
| X7 | **Medium** | panic handling | Panic handling is inconsistent and, where it exists, tends to fail open |
| X8 | **Medium** | security | `logger.Warn`/`Error` forward every message to Sentry unscrubbed, and error strings routinely embed attacker data *(partly fixed 2026-09-30: redaction and rate limiting added in `pkg/logger`; call sites still embed attacker data)* |
| X9 | **Low** | testing | Test coverage is extremely uneven: 5 packages have no test file at all |
The table is ordered by severity; the sections below are in ID order, since other
audit files reference these findings by number.
---
### X1. High — `-race` is never run
Verified by grep: the string `-race` does not appear in `Makefile`,
`.github/workflows/tests.yml`, `.github/workflows/maint.yml` or
`.github/workflows/make_tag.yml`.
Every test invocation in the repository:
```makefile
# Makefile:8
@go test ./pkg/resolvespec ./pkg/restheadspec -v -cover
# Makefile:13
@go test -tags=integration ./pkg/resolvespec ./pkg/restheadspec -v
# Makefile:97
@go test -tags=integration ./pkg/resolvespec ./pkg/restheadspec -v
# Makefile:103
@go test ./pkg/resolvespec ./pkg/restheadspec -coverprofile=coverage.out
# Makefile:110
@go test -tags=integration ./pkg/resolvespec ./pkg/restheadspec -coverprofile=coverage-integration.out
```
```yaml
# .github/workflows/tests.yml — unit-tests job
- name: Run unit tests
run: go test ./pkg/resolvespec ./pkg/restheadspec -v -cover
```
**Why this matters.** This audit found unsynchronized concurrent access to
mutable state in **six** packages, and the Go race detector would have flagged
every one of them on the first run:
| Package | Racing state | Reference |
|---|---|---|
| `pkg/cache` | `defaultCache` read/written by concurrent request handlers | `cache.audit.md` finding 3 |
| `pkg/config` | `*viper.Viper` has no internal lock; `configInstance` singleton | `config.audit.md` findings 1, 2 |
| `pkg/logger` | `Logger`, `errorTracker` globals | `logger.audit.md` finding 1 |
| `pkg/modelregistry` | `defaultRegistry` read by 6 functions without the lock | `modelregistry.audit.md` findings 2, 8 *(fixed 2026-09-30)* |
| `pkg/tracing` | `tracer` global | `tracing.audit.md` finding 5 *(fixed 2026-09-30)* |
| `pkg/errortracking` | `sentry.Init` mutates process globals | `errortracking.audit.md` finding 2 |
**Failure scenario.** `pkg/config` finding 1 is the sharpest illustration. A
concurrent `Manager.Set`/`Manager.Get` pair reaches viper's internal maps, which
have no mutex. A concurrent map read and write in Go is not a panic — it is
`fatal error: concurrent map read and map write`, which **`recover()` cannot
catch**. The process dies instantly, mid-request, with no graceful shutdown and
no error-tracker report. That is a remotely-triggerable hard crash, and it
cannot be found by inspection at scale — it is precisely what `-race` exists to
find. The detector has been in Go since 1.1 and costs one flag.
**Recommendation.** Add a race job that covers everything, and keep it separate
from the coverage run (race builds are ~2–10× slower):
```makefile
test-race:
@go test -race -count=1 ./pkg/...
```
```yaml
race-tests:
name: Race Detector
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v6
- uses: actions/setup-go@v6
with: { go-version: "1.24" }
- run: go test -race -count=1 ./pkg/...
```
Expect it to fail on the first run — that is the point. Fix `pkg/logger`,
`pkg/config` and `pkg/cache` first, since they are the shared dependencies. Note
that the race detector only reports races that **actually execute**, so X1 and X2
have to be fixed together: a race detector pointed at packages with no tests
finds nothing.
**Status (2026-09-30) — resolved for the packages with tests.** `make test-race` now exists
(`go test -race -count=1 ./pkg/...`), `test-unit` covers `./pkg/...`, and `test`
depends on both. The CI workflow (`.github/workflows/tests.yml`) now has a `race-tests`
job running the same command. The first full run was not clean:
| Package | Race | Kind |
|---|---|---|
| `pkg/logger` | `Logger` / `errorTracker` reassigned while other goroutines log (hit via `pkg/server` tests) | **production** — now guarded by an `RWMutex` (`getLogger`, `setLogger`, `getErrorTracker`); the exported `Logger` var is kept for compatibility |
| `pkg/security` | `DatabaseAuthenticator.Authenticate` passed `&userCtx` to the async session-activity goroutine while also returning it to the caller | **production** — the goroutine now gets a copy and is tracked by a `WaitGroup` so tests can wait for it |
| `pkg/security` tests | async activity update used sqlmock concurrently with the test adding expectations | test — tests wait via `authenticateSync` |
| `pkg/eventbroker`, `pkg/websocketspec` tests | handler/hook closures mutated a plain `bool`/`int` from worker goroutines | test — now `atomic` |
| `pkg/mqttspec` tests | not a race: hand-built `HookContext` lacked `TableName`/`Model`/`ModelPtr`, the unsubscribe test set `Data` instead of `SubscriptionID`, and `:memory:` SQLite gave each pooled connection its own empty database | test — fixed; these were failing without `-race` too |
`pkg/cache`, `pkg/config`, `pkg/modelregistry`, `pkg/tracing` and
`pkg/errortracking` are listed above but did **not** trip the detector: their
racing paths are not exercised by the current tests, which is the point made in
the paragraph above about X1 and X2 needing to be fixed together. Adding
concurrent tests for those globals is still outstanding.
Known limitation: `pkg/security` tests are not repeatable with `-count>1` (a
package-level capability cache carries over between runs), so the race target
keeps `-count=1`.
---
### X2. High — `go test` runs against 2 of 23 packages
Every `go test` invocation in the repository names exactly
`./pkg/resolvespec ./pkg/restheadspec`. No invocation uses `./...` or
`./pkg/...`.
The test bodies that exist but are never executed by CI:
| Package | Test files | Test lines | Run by CI? |
|---|---|---|---|
| `restheadspec` | 19 | 5 123 | **yes** |
| `resolvespec` | 8 | 2 379 | **yes** |
| `security` | 15 | 6 359 | no |
| `common` | 10 | 3 644 | no |
| `reflection` | 8 | 3 404 | no |
| `websocketspec` | 6 | 3 092 | no |
| `funcspec` | 3 | 2 416 | no |
| `spectypes` | 7 | 2 367 | no |
| `eventbroker` | 4 | 1 527 | no |
| `mqttspec` | 3 | 1 408 | no |
| `middleware` | 5 | 1 127 | no |
| `openapi` | 2 | 1 022 | no |
| `server` | 2 | 694 | no |
| `dbmanager` | 2 | 659 | no |
| `config` | 1 | 608 | no |
| `cache` | 1 | 69 | no |
| `errortracking` | 1 | 67 | no |
| `metrics` | 1 | 64 | no |
| `resolvemcp` | 1 | 34 | no |
| `logger` | 0 | 0 | — |
| `modelregistry` | 1 | ~150 | yes (`-race`) *(added 2026-09-30)* |
| `testmodels` | 0 | 0 | — |
| `tracing` | 1 | ~90 | yes *(added 2026-09-30)* |
**Failure scenario.** `pkg/security` has 6 359 lines of tests — the largest test
body in the repository — and **not one of them runs in CI**. A change that breaks
authentication, column-level security or row-security templates merges green.
The `maint.yml` job named "Run Vet Tests" is misleading: it runs `go mod
download`, `go mod verify` and `go vet ./...` and contains **no `go test` step at
all** (verified by grep). So the only signal on 21 of 23 packages is "it
compiles and vet is happy".
This directly explains the density of findings in this audit. The
`pkg/modelregistry` authorization fail-open (`modelregistry.audit.md` finding 1)
and the `pkg/cache`/`pkg/security` auth-outage-on-cache-failure
(`cache.audit.md` finding 1) are both the kind of defect a single unit test would
have caught, in packages that have never been tested.
**Recommendation.** Change every invocation to `./pkg/...`:
```makefile
test-unit:
@go test ./pkg/... -v -cover
```
```yaml
- name: Run unit tests
run: go test ./pkg/... -v -cover
```
If some currently-unrun package fails immediately, that is a bug report, not a
reason to keep the narrow list. Quarantine individual failing tests with
`t.Skip` and a `TODO` referencing an issue, so the *package* stays in the set.
---
### X3. High — integration failures cannot fail CI
`.github/workflows/tests.yml`, `integration-tests` job — every meaningful step
carries `continue-on-error: true`:
```yaml
- name: Run resolvespec integration tests
continue-on-error: true
env:
TEST_DATABASE_URL: "host=localhost user=postgres password=postgres dbname=resolvespec_test port=5432 sslmode=disable"
run: go test -tags=integration ./pkg/resolvespec -v -coverprofile=coverage-resolvespec-integration.out
- name: Run restheadspec integration tests
continue-on-error: true
...
```
**Failure scenario.** The integration suites are the only tests that exercise
real SQL generation against a real PostgreSQL — i.e. the only automated check on
the identifier-quoting and filter-construction paths that this audit's threat
model cares most about. Because both steps are `continue-on-error`, a SQL
injection regression, a broken join, or a total suite failure (wrong DSN, missing
migration) shows as a green check mark with a collapsed red step that nobody
opens. The job has no step that fails, so the job always passes. This is
strictly worse than not having the tests, because it creates the appearance of
coverage.
Note the integration DSN itself uses `sslmode=disable`, consistent with X6.
**Recommendation.** Remove `continue-on-error` from the two `go test` steps.
Keep it only on the coverage-report generation and artifact-upload steps, which
genuinely should not fail a build. If the suites are currently flaky, fix or
skip the flaky tests individually — `continue-on-error` on the whole step
disables the signal entirely.
---
### X4. Medium — `gosec` is not enabled — **RESOLVED**
> **Status (2026-09-30):** `gosec` is now in `linters.enable` and the repository lints clean
> (0 issues). The initial run produced 115 findings. Real fixes: login-form values in
> `security/oauth_server.go` are now HTML-escaped (G705), and `SqlSparseVector` index
> parsing uses `ParseInt(..., 10, 32)` (G109). The remaining ~110 sites carry
> `//nolint:gosec // Gxxx: <reason>` comments. The G201/G701 reasons (identifiers from
> trusted config or internal/validated names) and the G115 range claims were not
> individually audited and still need review. The text below describes the state before the change.
`.golangci.json` (`version: 2`) enables exactly three linters beyond the v2
standard set:
```json
"linters": {
"enable": [
"gocritic",
"misspell",
"revive"
],
```
golangci-lint v2's standard set (`errcheck`, `govet`, `ineffassign`,
`staticcheck`, `unused`) is on by default, so those do run. **`gosec` does not** —
it appears in the file only inside an exclusion rule for `_test.go`:
```json
{
"linters": [
"dupl",
"errcheck",
"gocritic",
"gosec"
],
"path": "_test\\.go"
},
```
Listing a linter in `exclusions.rules` does not enable it. The `lint` job in
`.github/workflows/maint.yml:40-57` does run golangci-lint over the whole
repository with `version: latest`, so the config is applied — it simply never
asks for the security checks.
**Failure scenario.** This repository builds SQL by string construction from
attacker-controlled schema, table, column and filter names (see
`restheadspec.audit.md` and `common.audit.md`). `gosec`'s `G201`/`G202`
(SQL string formatting/concatenation) are exactly the rules that would flag a
new `fmt.Sprintf` into a query, which is the single most likely way a SQL
injection enters this codebase. Also unenabled and relevant: `G104` (unhandled
errors — this audit found ~20 discarded errors in `pkg/cache` alone), `G304`
(file path from variable — relevant to `PathsConfig.Join`, `config.audit.md`
finding 14), `G402` (bad TLS settings — X6), `G404` (weak random).
**Recommendation.** Add `gosec` to `linters.enable` and triage the initial
findings. Expect noise on the SQL rules given the architecture; suppress
individual verified-safe sites with `//nolint:gosec // G201: identifier is
validated by X` comments that name the invariant, rather than disabling the rule
globally. That converts each suppression into a reviewable claim.
Consider also `bodyclose`, `rowserrcheck` and `sqlclosecheck` for a
database-heavy codebase, and `contextcheck` given how many methods here accept a
`ctx` and ignore it.
---
### X5. Medium — unsynchronized mutable package globals are the dominant pattern
Nine of the twenty-three packages expose mutable process-wide state through
package-level variables, and most guard it with nothing:
| Package | Global | Guarded? |
|---|---|---|
| `pkg/logger` | `Logger *zap.SugaredLogger` (`logger.go:15`), `errorTracker` (`:16`) | **no** — and `Logger` is exported |
| `pkg/cache` | `defaultCache *Cache` (`cache.go:10`) | **no** |
| `pkg/config` | `configInstance *Manager` (`manager.go:15`) | **no** |
| `pkg/tracing` | `tracer` | **yes** *(fixed 2026-09-30)* — `atomic.Pointer` |
| `pkg/modelregistry` | `defaultRegistry` | **yes** *(fixed 2026-09-30)* — guarded by `registriesMutex`; all access via `GetDefaultRegistry()` |
| `pkg/metrics` | `globalProvider` (`interfaces.go:50-51`) | **yes** — `globalProviderMu sync.RWMutex` |
`pkg/metrics` is the model the others should follow:
```go
// pkg/metrics/interfaces.go:50-72
var (
globalProviderMu sync.RWMutex
globalProvider Provider
)
func SetProvider(p Provider) {
globalProviderMu.Lock()
globalProvider = p
globalProviderMu.Unlock()
}
func GetProvider() Provider {
globalProviderMu.RLock()
p := globalProvider
globalProviderMu.RUnlock()
if p == nil {
return &NoOpProvider{}
}
return p
}
```
Note that it also returns a working `NoOpProvider` rather than `nil`, so callers
need no nil check — the pattern `pkg/logger` and `pkg/cache` should copy.
**Failure scenario.** Beyond the data races in X1, the shared failure mode is
**lazy initialization on the request path**. `cache.GetDefaultCache()`
(`cache.go:48`) and `config.GetConfigManager()` (`manager.go:18`) both
`if x == nil { x = construct() }` with no `sync.Once`. Under concurrent first
traffic, several instances are constructed and all but one are silently
discarded, so writes go to an orphaned object — a cache that is permanently 100%
miss, or two `Manager`s disagreeing about configuration. It presents as "the
cache doesn't work" with no error anywhere.
`pkg/logger.Logger` being **exported** and mutable is its own hazard: any
package, or any consumer of this library, can reassign the process logger
mid-flight while other goroutines are calling `Logger.Infow`.
**Recommendation.** For each global: `atomic.Pointer[T]` for
single-pointer swaps, `sync.Once` for lazy defaults, `sync.RWMutex` for
multi-field state. Unexport `logger.Logger` behind accessors. Where a nil global
is possible, return a no-op implementation instead of `nil`, as
`pkg/metrics.GetProvider` does.
---
### X6. Medium — insecure transport is the default everywhere
Every network dependency defaults to cleartext, and in two cases there is no way
to configure otherwise:
| Component | Default | Configurable? | Reference |
|---|---|---|---|
| PostgreSQL | `sslmode: disable` (`config/manager.go:242`) | yes, via config | `config.audit.md` finding 3 |
| OTLP traces | `otlptracegrpc.WithInsecure()` hardcoded (`tracing/tracing.go:41`) *(fixed 2026-09-30: TLS default, `Insecure` opt-in)* | **yes** | `tracing.audit.md` finding 1 |
| Redis (cache) | no `TLSConfig` set | **no** — `RedisConfig` has no TLS field | `cache.audit.md` finding 15 |
| Memcache | no TLS | **no** | `cache.audit.md` finding 15 |
| CORS | `allowed_origins: ["*"]`, `allowed_headers: ["*"]` (`config/manager.go:214-216`) | yes | `config.audit.md` finding 3 |
| DB user | `user: postgres` with blank password (`config/manager.go:239-240`) | yes | `config.audit.md` finding 3 |
The `tracing.go:41` case is the most pointed, because the code knows better:
```go
otlptracegrpc.WithInsecure(), // Use WithTLSCredentials in production
```
The comment names the fix and the config struct provides no way to apply it.
**Failure scenario.** The cache holds `UserContext` — identity and authorization
data — keyed by the raw bearer token (`security/providers.go:398`). With no TLS,
anything on the path between the service and Redis can read session contents and
the `AUTH` password, then **write** a forged `auth:session:<token>` entry.
`GetOrSet` returns a cache hit without consulting the database, so a forged entry
is a complete authentication bypass. Meanwhile the trace exporter ships full
request URLs including query strings (`tracing.audit.md` finding 2) in cleartext
to the collector.
**Recommendation.** Invert every default: TLS on unless explicitly disabled.
Concretely — add `TLS`/`TLSSkipVerify`/`TLSCACertFile` to `cache.RedisConfig`
and `tracing.Config`; change the `sslmode` default to `require`; change
`cors.allowed_origins` to `[]` and require an explicit list; remove the default
`postgres`/blank-password credentials so a misconfigured deployment fails to
start rather than connecting to a local database as a superuser. Add a startup
validation pass that logs a prominent warning for each insecure setting actually
in effect.
---
### X7. Medium — panic handling is inconsistent, and where it exists it fails open
Three different conventions coexist:
1. **`logger.CatchPanic(location)`** (`logger/logger.go:184`) — recovers, logs,
reports, and **swallows**. Both call sites are security enforcement:
`security/provider.go:302` (`ApplyColumnSecurity`) and `:443`
(`GetRowSecurityTemplate`). See `logger.audit.md` finding 4.
2. **`logger.HandlePanic(method, r)`** (`logger/logger.go:197`) — converts the
panic to an `error` the caller must handle. This is the correct shape.
3. **Nothing at all.** `pkg/cache` has zero `recover()` calls in 1 538 lines;
so do several other packages.
**Failure scenario (fail-open).** `ApplyColumnSecurity` panics — a nil map, a
bad type assertion on a rule, a reflection edge case. `CatchPanic` recovers and
the function returns normally, so the caller believes column security was
applied. It was not. The response contains the columns the security layer was
supposed to strip. The panic is logged, but the request succeeds with elevated
data exposure. A security control whose failure mode is "allow" is the wrong
default; it must be "deny".
**Failure scenario (panic under a lock).** `pkg/cache` holds `m.mu` across
`m.items[key] = ...` (`provider_memory.go:111`). After `Close()` sets
`items = nil` that assignment panics. With no recover in the package the panic
propagates to whatever handler exists upstream; if that handler recovers, `m.mu`
is **never unlocked** and every subsequent cache operation blocks forever. The
process stays alive and wedged — worse than a crash, because health checks that
do not touch the cache keep passing.
**Recommendation.** Establish one convention and apply it:
- **Request boundaries** (HTTP handlers, event consumers, goroutines): recover,
log with stack, report to the error tracker, return 500 / nack. A `go`
statement without a deferred recover is a process-kill waiting to happen —
`security/providers.go:447` (`go a.updateSessionActivity(...)`) is one.
- **Security enforcement**: recover, log, and **fail closed** — return an error
that the caller must propagate as a denial. Never `CatchPanic`.
- **Internal helpers**: do not recover. Let the boundary handle it.
- **Anything holding a lock**: prefer `defer mu.Unlock()` (already the pattern in
`pkg/cache`) so a panic cannot leak the lock, and keep panicking code out of
critical sections.
Add a `CatchPanicFailClosed(location string, err *error)` helper so the
fail-closed variant is as easy to reach for as `CatchPanic`.
---
### X8. Medium — attacker data reaches Sentry unscrubbed
Two facts compose badly:
`pkg/logger/logger.go:125-140` — every `Error` (and every `Warn`, `:108-123`)
forwards the fully-formatted message to the error tracker:
```go
func Error(template string, args ...interface{}) {
ctx, remainingArgs := extractContext(args...)
message := fmt.Sprintf(template, remainingArgs...)
...
if errorTracker != nil {
errorTracker.CaptureMessage(ctx, message, errortracking.SeverityError, map[string]interface{}{
"process_id": os.Getpid(),
})
}
}
```
And `pkg/errortracking` installs **no `BeforeSend` scrubber**
(`errortracking.audit.md` finding 1), so the message goes to Sentry verbatim.
Meanwhile error strings across the codebase interpolate attacker-controlled
values, sometimes secrets:
| Site | Interpolated value |
|---|---|
| `cache/cache_manager.go:26`, `:40` | the full cache key — for the session cache, **the raw bearer token** |
| `security/providers.go:391` | the raw `Authorization` header, logged at `Warn` when multiple tokens are present |
| `config/manager.go:164` | config file paths |
| throughout `restheadspec` | schema, table, column and filter values from the request |
**Failure scenario.** `security/providers.go:391` is live today:
```go
logger.Warn("Multiple authentication tokens provided in Authorization header (%d tokens). This is unusual and may indicate a misconfigured client. Header: %s", len(tokens), sessionToken)
```
A client sends two bearer tokens. `logger.Warn` formats the full header value
into the message and forwards it to Sentry, where a **valid session credential**
is now stored by a third party, visible to everyone with Sentry access, retained
per Sentry's policy, and replayable for the token's lifetime. No attacker
sophistication is required — the trigger is a single extra header, and the
codebase invites it by logging the header contents as the diagnostic.
**Recommendation.**
1. Add a `BeforeSend` hook in `pkg/errortracking` that redacts
`Authorization`, `Cookie`, `Set-Cookie`, anything matching
`(?i)(token|password|secret|apikey|api_key|bearer)\s*[:=]\s*\S+`, and
long high-entropy strings. This is the one change that bounds the whole class.
2. Never log a credential, even truncated. Change `providers.go:391` to log
`len(tokens)` only.
3. Replace `fmt.Errorf("key not found: %s", key)` with a sentinel
`cache.ErrNotFound` (`cache.audit.md` finding 7).
4. Key the session cache on `sha256(token)`, as
`security/keystore_database.go:287` already does for API keys.
5. Add sampling / rate limiting to the tracker fan-out
(`logger.audit.md` finding 3) so an error storm is not also a cost and
availability event.
---
### X9. Low — five packages have no tests at all
`pkg/logger`, `pkg/modelregistry`, `pkg/testmodels`, `pkg/tracing` have zero
`*_test.go` files. `pkg/resolvemcp` has 34 lines, `pkg/metrics` 64,
`pkg/errortracking` 67, `pkg/cache` 69.
**Failure scenario.** `pkg/modelregistry` is untested and contains this audit's
only **Critical** authorization finding: `GetModel` returns a "registry locked"
error under write-lock contention, which `security/hooks.go:274-294` converts
into `return nil // model not registered, allow by default`
(`modelregistry.audit.md` finding 1; *fixed 2026-09-30, regression tests added*). A twenty-line test that registers a model
from one goroutine while reading it from another would demonstrate the fail-open
immediately. The package guards a security boundary and has never been tested.
`pkg/logger` being untested matters for a different reason: it is imported by
almost every other package, so a defect there (the format-string sink in `Info`
and `Debug`, `logger.audit.md` finding 6) is repo-wide.
**Recommendation.** Prioritize by blast radius, not by size:
1. `pkg/modelregistry` — concurrent register/read; assert `GetModelRulesByName`
never returns a "locked" error that a caller could read as "not registered".
2. `pkg/logger` — nil-`Logger` fallback paths, format-string handling, and that
`Warn`/`Error` do not forward secrets once a scrubber exists.
3. `pkg/cache` — concurrent `GetDefaultCache`, the expired-item TOCTOU, and that
`tagToKeys` does not grow after eviction.
4. `pkg/tracing`, `pkg/metrics`, `pkg/errortracking` — construction and no-op
paths; these are mostly configuration surfaces.
Combine with X1 and X2: tests that are not run, and tests run without `-race`,
do not close these gaps.
---
### X10. High — configured subsystems that are never installed
Three separate subsystems are fully built — typed config, defaults, tests,
documentation — and then never connected to anything that runs.
**1. Every protective middleware.** `pkg/middleware` provides rate limiting, IP
blacklisting, request-size limiting and input sanitization. Non-test callers:
| Constructor | Non-test callers |
|---|---|
| `middleware.NewRateLimiter` | **0** |
| `middleware.NewIPBlacklist` | **0** |
| `middleware.NewRequestSizeLimiter` | **0** |
| `middleware.DefaultSanitizer` | **0** outside the package |
| `middleware.StrictSanitizer` | **0** |
| `middleware.PanicRecovery` | 1 — `pkg/server/manager.go:466` |
`pkg/server/manager.go` is the only file outside the package that imports it, and
only for `PanicRecovery`. The config that exists to drive the rest —
`MiddlewareConfig.RateLimitRPS`, `.RateLimitBurst`, `.MaxRequestSize`
(`pkg/config/config.go:123-125`), defaulted at `pkg/config/manager.go:209-211` —
has **no reader anywhere in the module**.
**2. The metrics provider.** `metrics.SetProvider` and
`metrics.NewPrometheusProvider` have **0 non-test callers**, so
`metrics.GetProvider()` returns `&NoOpProvider{}`
(`pkg/metrics/interfaces.go:63-72`) for the process lifetime. Every instrumented
call site in the repository — 39 DB-query sites in
`pkg/common/adapters/database`, the HTTP middleware, the event-broker counters,
and the sole `RecordPanic` call at `pkg/middleware/panic.go:19` — writes to a
no-op. `MetricsConfig.Enabled` and `.Provider` are likewise never read, and
`pkg/config` has no `metrics` section at all.
**3. The configured CORS policy.** `config.CORSConfig`
(`pkg/config/config.go:128-134`), defaulted at `pkg/config/manager.go:214-217`,
is never read. The policy that actually applies comes from a **different type of
the same name**, `common.CORSConfig`, built by `common.DefaultCORSConfig()`
(`pkg/common/cors.go:19-48`), which derives allowed origins from the configured
server instances and the host's local IPs and ignores `cors.allowed_origins`
entirely. It is called from ten sites across `pkg/resolvespec` and
`pkg/restheadspec`.
**Failure scenario.** Each of these is a silent, config-shaped lie, and they fail
in the same way: the operator's mental model of the deployment is wrong in the
direction of believing a control exists.
- **Under the hostile-client threat model there is no rate limit and no
request-body limit in the serving path.** `max_request_size: 10485760` is
configured and unenforced, so a single unauthenticated `POST` with a
multi-gigabyte body is read into memory and OOM-kills the process; unlimited
request rate exhausts the 25-connection default pool
(`pkg/config/manager.go:224`) just as cheaply. Both are one-line attacks
against controls the configuration says are active. An operator lowering
`rate_limit_rps` during an incident observes no change and will reasonably
conclude the attack exceeds the limit rather than that no limit exists.
- **There is no telemetry with which to notice any of it.** No request counts, no
latency histograms, no `panics_total`, no DB-query metrics — the one signal
that would show an attack in progress is wired end to end and discarded at the
last step. This is also why the metrics cardinality defects
(`metrics.audit.md` findings 2 and 5) are only latent: they become live the
moment someone installs the provider that the config implies is already there.
- **Tightening `cors.allowed_origins` does nothing.** The value is ignored, so a
hardening change lands, reviews clean, deploys, and changes no behaviour. Two
types named `CORSConfig` in two packages is the mechanism; nothing warns.
The common thread is that none of this fails visibly. It compiles, the tests pass
(`pkg/middleware` has the repo's best test ratio — 1 127 test lines to 799 code
lines — all of it exercising code nothing calls), CI is green, and the config file
documents features that are absent. Under X2 these packages are not even in the
tested set, so the tests that do exist are not run.
**Recommendation.**
1. **Wire the middleware chain** in `pkg/server` from `MiddlewareConfig`,
outermost first: size limiter → rate limiter → blacklist → `PanicRecovery`
(innermost, so it sees handler panics; `trackRequestsMiddleware` at
`manager.go:540` correctly stays outside). Fix the trusted-proxy handling
(`middleware.audit.md` findings 2 and 3) **before** mounting the two IP-based
layers, and do not mount the sanitizer at all until findings 5–7 there are
resolved — as written it corrupts filter values and can synthesize a
`javascript:` URI.
2. **Install a metrics provider** from config, gated on `metrics.enabled`, and
add the missing `metrics` section to `pkg/config`. Bound the label sets first
(`metrics.audit.md` findings 2 and 5) — installing the provider as-is converts
two latent cardinality DoS findings into live ones.
3. **Delete the duplicate `CORSConfig`** or make `common.DefaultCORSConfig()`
read `config.CORSConfig`. Two types with one name, one of them ignored, is a
trap regardless of which way it is resolved.
4. **Make the class of defect detectable.** Log at startup which middleware,
metrics provider and CORS policy are active, so an unwired subsystem is
visible in the first ten lines of a boot log instead of during an incident.
A CI check that every `mapstructure` field in `pkg/config` has at least one
reader would have caught all three of these; so would enabling `unused` in
`.golangci.json` for exported-but-unreferenced constructors.
---
## Recommended order of work
1. **X2 + X1** — point `go test` at `./pkg/...` and add a `-race` job. Everything
else in this audit is easier to verify once these exist, and they will
surface the six data races on their own.
2. **X3** — remove `continue-on-error` from the integration `go test` steps.
3. **X10** — mount the request-size limiter and rate limiter. Until this is
done the service has no volumetric protection at all, and no metrics with
which to see that. Fix `middleware.audit.md` findings 2 and 3 in the same
change, since mounting the IP-based layers without them adds attack surface.
4. **X8 item 1 and 2** — add the Sentry `BeforeSend` scrubber and stop logging
the `Authorization` header. Small, self-contained, stops an active credential
leak.
5. **X7** — decide the panic convention; make the two `CatchPanic` sites in
`pkg/security` fail closed, and stop returning the panic value to the client
(`pkg/middleware/panic.go:28`).
6. **X5** — fix the globals in `pkg/logger`, `pkg/config`, `pkg/cache`
(the shared dependencies) first.
7. **X6** — add TLS fields and invert the defaults.
8. **X4** — enable `gosec` and triage.
9. **X9** — backfill tests, in the order listed above.
## Per-package audits
`cache` · `common` · `config` · `dbmanager` · `errortracking` · `eventbroker` ·
`funcspec` · `logger` · `metrics` · `middleware` · `modelregistry` · `mqttspec` ·
`openapi` · `reflection` · `resolvemcp` · `resolvespec` · `restheadspec` ·
`security` · `server` · `spectypes` · `testmodels` · `tracing` · `websocketspec`
Each is `audit/pkg/<name>.audit.md`.
File diff suppressed because it is too large Load Diff
+407
View File
@@ -0,0 +1,407 @@
# Audit: `pkg/common`
| | |
|---|---|
| **Package** | `github.com/bitechdev/ResolveSpec/pkg/common` (+ `adapters/database`, `adapters/router`) |
| **Files** | `sql_helpers.go` (1060), `recursive_crud.go` (645), `validation.go` (444), `json_column.go` (402), `interfaces.go` (311), `spatial_helpers.go` (317), `handler_utils.go` (309), `json_condition.go` (219), `types.go` (192), `cors.go` (156), `handler_example.go` (97); `adapters/database/bun.go` (1767), `pgsql.go` (1600), `gorm.go` (1018), `query_metrics.go` (335), `pgsql_preload_example.go` (275), `pgsql_example.go` (176), `test_helpers.go` (132), `utils.go` (117); `adapters/router/mux.go` (238), `bunrouter.go` (214) |
| **Audit date** | 2026-09-30 |
| **Axes** | thread locking/waiting, slowness, security, panic handling & logging |
| **Threat model** | hostile internet client; request bodies, headers, query params, schema/table/column names all attacker-controlled |
| **Depth** | deep for `sql_helpers.go`, `validation.go`, `cors.go`, `recursive_crud.go`, `json_column.go`/`json_condition.go` and the reconnect/transaction paths of the three DB adapters; medium for the rest; the `*_example.go` files were skimmed. Findings 1–3 were verified with throw-away probe tests, which were deleted afterwards |
## Summary
`pkg/common` is the shared core behind every spec handler. It contains the
`Database` / `SelectQuery` abstraction and its Bun, GORM and raw-`pgx`
adapters, the request-option types, column validation, JSON-column parsing,
nested (recursive) CRUD, CORS, and a set of SQL string helpers. The spec
packages feed **client-supplied raw SQL fragments** through those helpers:
`x-custom-sql-w`, `x-custom-sql-or`, `x-custom-sql-join`, preload `where`,
sort expressions and cursor filters.
The central problem is that **`SanitizeWhereClause` / `validateWhereClauseSecurity`
is a keyword denylist applied to raw SQL**, and the result is concatenated
straight into the query. A denylist can't make arbitrary client SQL safe, and
this one misses subqueries, functions, comments and parenthesis balancing.
With the helpers exactly as the handlers call them, a client can:
- escape the outer parentheses and OR past every filter the server adds
afterwards (**row security, tenant filters, the PK filter**). This is
verified. Row security is inert anyway today (`security.audit.md` finding 2),
but this bug will defeat it as soon as that is fixed;
- read any table the DB role can see through a subquery;
- stall a connection with `pg_sleep`;
- bypass the keyword list with a comment (`delete/**/from`).
Meanwhile, legitimate filters that merely *contain* a word like `update` are
silently dropped, and the query runs **unfiltered** (fail-open).
Other headline findings:
- `SetCORSHeaders` reflects **any** Origin with `Allow-Credentials: true` and
ignores `AllowedOrigins`.
- Nested CRUD updates and deletes child rows by primary key alone. A client can
modify or delete (or re-parent) any row in a related table.
- Sort validation lets arbitrary SQL through whenever a custom join has no
alias.
On the question that started this audit (idle connections becoming unusable),
the relevant part of `pkg/common` is the adapters' reconnect logic (finding 5).
It is only partly wired into the Bun and pgx adapters, and it's what calls
`dbmanager`'s destructive `Reconnect` (`dbmanager.audit.md` findings 1–2).
## Findings
| # | Severity | Axis | Finding |
|---|---|---|---|
| 1 | **Critical** | security | Client raw-SQL WHERE (`x-custom-sql-w`/`-or`, preload where, cursor) is protected only by a keyword denylist: parenthesis escape defeats server-added filters; subqueries, `pg_sleep` and comment bypasses all pass (verified) |
| 2 | **Critical** | security | `SetCORSHeaders` reflects any `Origin` and sends `Access-Control-Allow-Credentials: true`; `AllowedOrigins` is never consulted |
| 3 | **High** | security | Sort validation: an empty join alias makes `strings.Contains(col, "")` accept **any** sort string, and `(…)` sort expressions allow arbitrary subqueries (verified) |
| 4 | **High** | security | Nested CRUD (`recursive_crud.go`) updates and deletes child rows by `WHERE pk = ?` only, with no parent/ownership constraint; a client-supplied `_request` switches the operation per object |
| 5 | **High** | locking / availability | Adapter reconnect is inconsistent (Bun and pgx query builders never reconnect) and, where it exists, calls dbmanager's pool-closing `Reconnect`; `BunAdapter.CommitTx`/`RollbackTx` are silent no-ops |
| 6 | **Medium** | security / correctness | `SanitizeWhereClause` fails open: on a denylist hit it returns `""`, so the client's filter is dropped and the query returns unfiltered rows; false positives on ordinary data (`'awaiting update'`, `last_update`) |
| 7 | **Medium** | security | Any column name starting with `cql` passes `ValidateColumn` unconditionally |
| 8 | **Medium** | slowness | Request bodies are read with unbounded `io.ReadAll` in both router adapters |
| 9 | **Medium** | logging | Failed queries log the fully interpolated SQL, and nested CRUD logs full row data, at `Error`, which is forwarded to Sentry (`_CROSS-CUTTING.audit.md` X8) |
| 10 | **Low** | correctness | `stripEmptyComparisonClauses` regexes rewrite SQL without respecting string literals; quote tracking ignores `''`; `qualifyColumnInCondition` compiles a regex per call |
| 11 | **Low** | locking | Adapter fields read without their mutex (`BunAdapter.NewSelect` `db: b.db`, `DriverName`, `PgSQLAdapter.GetUnderlyingDB`) race with `reconnectDB` |
| 12 | **Info** | — | `json_column.go` / `json_condition.go` are well built: allowlisted casts, path bound as a single `text[]` parameter, identifiers validated and quoted |
---
### 1. Critical — Client raw-SQL WHERE is guarded only by a keyword denylist
`sql_helpers.go:118-161` (`validateWhereClauseSecurity`), `169-305`
(`SanitizeWhereClause`), `375-395` (`EnsureOuterParentheses`).
Call sites that pass **client-controlled** strings:
| Source | Call site |
|---|---|
| `x-custom-sql-w` | `restheadspec/handler.go:692-699` → `query.Where(...)` |
| `x-custom-sql-or` | `restheadspec/handler.go:703-710` → `query.WhereOr(...)` |
| `x-custom-sql-join` | `restheadspec/headers.go:666`, `1330` (sanitized with `tableName ""`) |
| preload `where` | `resolvespec/handler.go:2386, 2458`; `restheadspec/handler.go:618, 1170` |
| cursor filters | `resolvespec/handler.go:458`; `restheadspec/handler.go:916`; `resolvemcp/handler.go:304` |
The pipeline is `AddTablePrefixToColumns` → `SanitizeWhereClause` →
`EnsureOuterParentheses` → `query.Where(s)`, with **no bind arguments**. The
only security check is a substring search for `delete `, `update `, `drop `,
`;delete`, and similar.
A probe reproduced the handler pipeline and then appended a server-side filter
`Where("tenant = ?", 5)`, which is what a row-security or tenant hook does:
| Client `x-custom-sql-w` | Resulting SQL / effect |
|---|---|
| `1=1)) OR ((1=1` | `WHERE ((1=1)) OR ((1=1)) AND (tenant = 5)`: **every tenant's rows**, because `AND` binds tighter than `OR` |
| `id = 1 or (select count(*) from pg_shadow) > 0` | passes unchanged, so boolean-oracle exfiltration from any readable table works |
| `id = 1 and pg_sleep(5) is not null` | passes; each request pins a pool connection for as long as the client likes |
| `id = 1; delete/**/from items` | passes, because the comment defeats `"delete "` (whether it executes depends on the driver's multi-statement handling) |
`EnsureOuterParentheses` only checks whether the string *already* starts and
ends with a matching pair. It never checks that the parentheses inside are
balanced, which is what the escape relies on. `x-custom-sql-or` is worse by
design: `WhereOr` ORs the client clause against **every** condition already on
the query, so it needs no escape at all to widen a server-side filter.
Row security currently has no effect (`security.audit.md` finding 2). Fixing
that type assertion will **not** give tenant isolation while these headers
exist, and the same escape defeats the server's own PK scoping
(`restheadspec/handler.go:759-766`).
**Failure scenario.** An authenticated user of tenant A sends
`X-Custom-SQL-W: 1=1)) OR ((1=1` on a list endpoint and receives tenant B's
rows. Or they send
`X-Custom-SQL-W: (select substr(passwd,1,1) from pg_shadow limit 1) = 'm'` and
extract data one character at a time.
**Recommendation.** Stop accepting raw SQL from clients. Remove the
`x-custom-sql-*` headers from the public surface, or gate them behind an
explicit server-side allowlist per endpoint. Route client filtering through
the structured `FilterOption` path, which validates column names and binds
values. If raw fragments have to stay for trusted internal callers:
- parse them properly (for example with `pg_query_go`) and allow only column
references, literals and comparison operators;
- reject subqueries and function calls;
- verify that parentheses are balanced outside string literals;
- apply server-side security predicates last, as a wrapper
`WHERE (server) AND (client)`, and never let `WhereOr` attach at top level.
---
### 2. Critical — CORS reflects every Origin with credentials
`cors.go:117-155`:
```go
origin := r.Header("Origin")
if origin == "" {
origin = "*"
} else { ... Vary: Origin }
w.SetHeader("Access-Control-Allow-Origin", origin)
...
requestedHeaders := r.Header("Access-Control-Request-Headers")
if requestedHeaders != "" {
w.SetHeader("Access-Control-Allow-Headers", requestedHeaders)
}
...
if origin != "*" {
w.SetHeader("Access-Control-Allow-Credentials", "true")
}
```
`DefaultCORSConfig` (`cors.go:19-48`) carefully builds `AllowedOrigins` from the
server config, and `SetCORSHeaders` **never reads it**. Any site the victim
visits can make credentialed cross-origin requests and read the responses.
Allowed request headers are also reflected, so `Authorization` and every
`X-Custom-SQL-*` header pass preflight. `SetCORSHeaders` is called on every
route in `resolvespec/resolvespec.go` (lines 56-347), and `restheadspec` follows
the same pattern.
**Failure scenario.** A user logged in through cookie auth (`SetSessionCookie`/`GetSessionCookie`,
`pkg/security/middleware.go:512-540`) visits `evil.example`. Its script calls
`fetch("https://api/…/users", {credentials:"include"})` and reads every record
the user can see. Combined with finding 1, it can read other tenants' records
too.
**Recommendation.** Send `Allow-Origin: <origin>` and `Allow-Credentials`
only when `origin` is in `config.AllowedOrigins`, matched exactly. Otherwise
omit the CORS headers. Check requested headers against `AllowedHeaders`
instead of echoing them. Build `exposeHeaders` in a fresh slice: `append` onto
`config.AllowedHeaders` can write into a shared backing array.
---
### 3. High — Sort validation bypasses
`validation.go:271-301`:
```go
foundJoin := false
for _, j := range options.JoinAliases {
if strings.Contains(sort.Column, j) { // j may be ""
```
`restheadspec/headers.go:674-678` deliberately appends `""` to `JoinAliases`
when `extractJoinAlias` can't find an alias (for example
`LEFT JOIN t ON …` with no alias, or a LATERAL join without one).
`strings.Contains(x, "")` is always `true`, so **any** sort string is
accepted. `restheadspec/handler.go:790-793` then passes anything containing a
`.` or wrapped in `(…)` to `OrderExpr` **verbatim**. Even with a real alias,
the check is a substring test, so a sort like `j.id, (select …)` passes for
alias `j`.
Separately, `(…)` sort expressions are checked by `IsSafeSortExpression`
(`validation.go:381-427`), another denylist. It blocks DML keywords, comments
and `;`, but allows subqueries and functions.
Probe results: with `JoinAliases: [""]`, sort `x.id, (select pg_sleep(10))`
was kept. With no joins, sort `(select passwd from pg_shadow limit 1)` was kept.
**Failure scenario.** A client sends a custom join with no alias plus an
arbitrary ORDER BY expression, which gives injection in ORDER BY: time-based
DoS, or data extraction via `ORDER BY (CASE WHEN (subquery) THEN a ELSE b END)`.
**Recommendation.** Skip empty aliases. Match `alias + "."` as a prefix, then
validate the column after the dot against the joined table. Drop client
supplied sort *expressions*, or restrict them to a server-registered set
(the `cql` computed columns already provide this).
---
### 4. High — Nested CRUD modifies arbitrary related rows
`recursive_crud.go:64-67, 144-196, 344-380, 395-520`.
- Children are updated with `UPDATE <related> SET … WHERE pk = ?`, deleted with
`DELETE FROM <related> WHERE pk = ?`, and both use the child's PK **from the
request body**. Nothing checks that the child belongs to the parent being
written or to the caller's tenant.
- For updates, the parent's FK is injected into the child data
(`recursive_crud.go:495-520`), so updating a foreign child **moves it under
the attacker's parent** as well.
- `_request` (`recursive_crud.go:64-67, 205-212`) lets the client choose
`insert`/`update`/`delete` for each nested object, independent of the HTTP
method or the operation the top-level handler authorised.
- These statements go straight to `p.db`, so the spec handlers' Before*/After*
hooks, and any row-security or audit hooks, don't run for nested rows.
**Failure scenario.** A client sends a `PUT /orders/1` whose body includes
`"lines": [{"id": 9999, "_request": "delete"}]`. Row 9999 of `order_lines` is
deleted even if it belongs to another customer's order.
**Recommendation.** For has-many and has-one children, add
`AND <fk> = <parentID>` to update and delete statements, and treat
`RowsAffected() == 0` as a forbidden or not-found error. Run the same hook
chain (including row security) for nested rows. Allow `_request` only for
operations the top-level request is authorised to perform.
---
### 5. High — Reconnect logic is partial, and it triggers pool destruction
`adapters/database/bun.go:131-143, 167-186, 226-273, 1298-1318`;
`pgsql.go:58-79, 81, 134, 160, 220`; `gorm.go:55, 122-134`.
- **Coverage is uneven.** `BunAdapter` only retries after reconnecting in
`Exec`, `Query`, `BeginTx` and `RunInTransaction`. `NewSelect`, `NewInsert`,
`NewUpdate` and `NewDelete` capture `getDB()` once, and `BunSelectQuery.Scan`,
`ScanModel`, `Count` and `Exists` call bun directly with no retry. Those are
the paths every read handler uses. `PgSQLAdapter` query builders have no
reconnect either. Only `GormAdapter` wires `reconnect` into its
select, insert, update and delete builders.
- **Where it exists, it's harmful.** `reconnectDB` calls the dbmanager factory,
which runs `sqlConnection.Reconnect` and closes the pool shared by every
other adapter and handle (`dbmanager.audit.md` findings 1–2). Concurrent
failures each call the factory.
- **Detection is a substring match.** `isDBClosed` (`pgsql.go:72`) matches
`"sql: database is closed"`. That only happens *after* someone closed the
pool, so the reconnect mechanism mainly exists to recover from damage it
causes itself. It does nothing for the real idle-socket failure (a hang, or
`driver.ErrBadConn`, which `database/sql` already retries).
- `BunAdapter.CommitTx` / `RollbackTx` (`bun.go:239-249`) return `nil` without
doing anything. A caller using the `BeginTx`-less path gets
"committed" when nothing happened. `BunTxAdapter` is correct.
**Recommendation.** Remove adapter-level reconnect entirely and rely on
`database/sql`'s pool (see the fix order in `dbmanager.audit.md`). Make
`BunAdapter.CommitTx`/`RollbackTx` return an explicit
"not in a transaction" error. Add a per-query `context.WithTimeout` in the
adapters as the single place where query deadlines are enforced.
---
### 6. Medium — `SanitizeWhereClause` fails open and has false positives
`sql_helpers.go:176-179`:
```go
if err := validateWhereClauseSecurity(where); err != nil {
logger.Debug("Security validation failed for WHERE clause: %v", err)
return ""
}
```
Every caller treats `""` as "no filter" and skips `query.Where`. So a clause
the sanitizer rejects is **removed**, and the request runs unfiltered instead
of failing. The denylist is a substring match on the whole clause, string
literals included, so ordinary filters trip it. The probe showed
`status = 'awaiting update approval'` and `last_update > '2020-01-01'` both
returning `""`, which gives an unfiltered list. For a preload `where` or a
cursor filter, that means returning rows the client asked to exclude, or
breaking pagination.
**Recommendation.** Return an error and make the handler respond `400`. Never
turn a rejected filter into "no filter".
---
### 7. Medium — `cql*` columns bypass column validation
`validation.go:107-110` accepts any column that starts with `cql`
(case-insensitive), with no further checks. The probe showed
`IsValidColumn("cql1); drop")` returning `true`. The computed-column mechanism
only ever generates `cql1…cqlN` (`restheadspec/headers.go:871, 1380`). Whether a
client-supplied `cql…` string reaches SQL unquoted depends on the downstream
handler (`restheadspec/handler.go:490-509`, `cursor.go:189`). The validator
shouldn't be where that decision is made.
**Recommendation.** Accept only `^cql[0-9]+$`, and only when that computed
column was actually registered for the request.
---
### 8. Medium — Unbounded request body reads
`adapters/router/mux.go:101-115` uses `io.ReadAll(h.req.Body)` with no
`http.MaxBytesReader`, and `bunrouter.go:102-115` delegates to it. The
request-size middleware exists but isn't mounted (`_CROSS-CUTTING.audit.md`
X10), so a single request can make the process buffer gigabytes.
**Recommendation.** Wrap the body in `http.MaxBytesReader` inside the adapter,
with a configurable limit (for example 10 MB) and a sensible default.
---
### 9. Medium — Sensitive data in error logs
- `bun.go:1311-1315` (and the equivalent in `ScanModel`/`Count`, and in
`pgsql.go` / `gorm.go`) logs `b.query.String()`, the SQL with **all argument
values interpolated**, at `Error` on every failed query. That includes
filter values, emails, and tokens used as lookup keys.
- `recursive_crud.go` logs `data=%+v` (whole rows, including password or secret
columns) at `Error` on every failed nested write (lines 121, 153, 312, 342,
352, 509, 528, 550).
- `logger.Error` is forwarded to Sentry unscrubbed (`_CROSS-CUTTING.audit.md`
X8), and a hostile client can trigger failing queries at will.
**Recommendation.** Log the query with placeholders, not interpolated. Log
column names, not values. Put full dumps behind a debug flag.
---
### 10. Low — Fragile SQL string rewriting
- `reEmptyCompMid` / `reEmptyCompEnd` (`sql_helpers.go:66-80`) run over the
whole SQL string, including string literals and subqueries, and silently
delete text that matches `col = and`. That can change a query's meaning.
- The quote tracking in `splitByAND` / `findOperatorOutsideParentheses` /
`stripWrappingParens` toggles on every `'`, so an escaped `''` inside a
literal flips the state.
- `qualifyColumnInCondition` (`sql_helpers.go:751-760`) compiles a regex on
every call in the per-request path.
---
### 11. Low — Unsynchronised adapter field reads
`BunAdapter` protects `db` with `dbMu` in `getDB`/`reconnectDB`. But
`NewSelect` stores `db: b.db` (`bun.go:170`, used for count queries), and
`DriverName` reads `b.db` (`bun.go:280`), both without the lock.
`PgSQLAdapter.GetUnderlyingDB` (`pgsql.go:220`) does the same. These are data
races with `reconnectDB`, and `-race` would flag them
(`_CROSS-CUTTING.audit.md` X1). In practice the count query can run against
the old, closed pool.
---
### 12. Info — JSON column parsing is sound
`json_column.go` / `json_condition.go` are a good model for how the rest of
this package should handle client input:
- The base column must match `^[A-Za-z_][A-Za-z0-9_]*$` and is quoted with
`QuoteIdent`.
- Casts go through an allowlist.
- The JSON path is always bound as one `?::text[]` parameter, with depth and
segment-size limits.
- The dotted shorthand counts as JSON only when reflection confirms that the
base is a JSON column.
- The alias is validated and quoted.
No findings.
---
## Panic handling
The adapter methods (`Scan`, `ScanModel`, `Count`, `Exec`, `Query`,
`RunInTransaction`) recover and convert panics with `logger.HandlePanic`.
`PgSQLAdapter.RunInTransaction` rolls back on panic before re-raising or
converting it. `BunAdapter.RunInTransaction` relies on bun's `RunInTx`, which
also rolls back. No panic paths were found in `sql_helpers.go`,
`validation.go` or the JSON parser that are reachable from client input.
Slicing is length-guarded. `recursive_crud.go` recurses over the model's
relation graph. For self-referential models, depth is bounded only by the
JSON decoder's nesting limit, which makes it a slowness issue rather than a
crash.
## Test coverage
`sql_helpers_test.go` and `validation_test.go` test the *intended* behaviour
of the sanitizer and validator. None of them test hostile inputs. Each probe
case in findings 1, 3, 6 and 7 is a one-line table entry and should be added
as a regression test that asserts rejection. `cors.go` and `recursive_crud.go`
have no security-focused tests.
+419
View File
@@ -0,0 +1,419 @@
# Audit — `pkg/config`
- **Date:** 2026-09-29
- **Scope:** `pkg/config/{config,dbmanager,manager,paths,server}.go` (1023 LOC source, 608 LOC tests)
- **Axes:** thread locking/waiting · slowness · security · panic handling & logging
- **Threat model:** hostile internet client. Config itself is operator-controlled, so the security
focus here is **insecure defaults that the internet-facing layers inherit**, plus secret handling.
## Summary
Viper-backed configuration with a singleton `Manager`, a large `setDefaults` table, and per-section
validators. Two serious issues:
1. **`Manager` is a data race by construction.** It wraps a `*viper.Viper`, which has **no internal
locking** (verified: no `sync.Mutex`/`RWMutex` anywhere in `viper@v1.21.0/viper.go`'s `Viper`
struct), and exposes `Get`/`Set` as concurrently-callable methods on an unsynchronised lazy
singleton. A concurrent `Set` + `Get` is a concurrent map write → **`fatal error`, not a
recoverable panic**.
2. **The default configuration is insecure on every axis that matters** — wildcard CORS,
`sslmode=disable`, `user: postgres` with a blank password — and `Load()` silently succeeds when
no config file is found, so a misdeployment lands on exactly those defaults with no warning.
| # | Severity | Axis | Finding |
|---|----------|------|---------|
| 1 | **Critical** | Locking | `Manager.Set`/`Get` over a lock-free `*viper.Viper` → concurrent map write → process-fatal |
| 2 | **High** | Locking | `GetConfigManager()` is an unsynchronised lazy singleton; `NewManager()` also clobbers the global as a side effect |
| 3 | **High** | Security | **OPEN (deferred)** Insecure defaults: `cors.allowed_origins: ["*"]`, `allowed_headers: ["*"]`, `sslmode: disable`, `user: postgres` + blank password |
| 4 | **High** | Security | `SaveConfig` writes all secrets in plaintext at mode `0644` (viper default, never overridden) |
| 5 | Medium | Security | `AddConfigPath(".")` is searched first — CWD config injection |
| 6 | Medium | Observability | `Load()` swallows `ConfigFileNotFoundError` with no log at all |
| 7 | Medium | Correctness | `PathsConfig.Set` on a nil map panics; every sibling method nil-guards |
| 8 | Medium | Locking | `PathsConfig` is a bare `map[string]string` with a mutating `Set` — concurrent access is process-fatal |
| 9 | Medium | Slowness | `GetIPs()` does an uncontexted `net.LookupIP` — blocks on the resolver timeout |
| 10 | Medium | Correctness | `SetConfig` does a pointless `Unmarshal` into a discarded map whose error fails the call |
| 11 | Low | Panic | `GetIPs()` recovers to `fmt.Println`, bypassing the logger, and returns zeroed named results |
| 12 | Low | Security | No validation of `middleware.*` / `event_broker.worker_count` — `0` workers is accepted |
| 13 | Low | Correctness | `ServersConfig.GetDefault()` returns a pointer to a copy of a map value |
| 14 | Low | Security | `PathsConfig.Join` does not confine the result to the base path |
## Resolution status (2026-09-30)
- **#1** — Fixed: `sync.RWMutex` guards every viper access, options included
- **#2** — Fixed: mutex-guarded singleton; `NewManager` no longer touches the global (new `SetConfigManager` publishes explicitly)
- **#4** — Fixed: `SetConfigPermissions(0o600)` plus `chmod 0600` after write (secrets are not stripped)
- **#5** — Fixed: search order is `/etc/resolvespec`, `$HOME/.resolvespec`, `./config`, `.` (CWD last, not dropped)
- **#6** — Partly fixed: `ConfigFileUsed()` added; no log line because `pkg/config` cannot import `logger` (import cycle)
- **#7** — Fixed: `Set` has a pointer receiver and allocates
- **#8** — Not fixed: still a bare map; `Set` documented as not concurrency-safe
- **#9** — Fixed: `LookupIPAddr` with a 2s timeout, fallback normalised to bare IPs and populates the slice
- **#10** — Fixed: dead `Unmarshal` removed, `SetConfig` is atomic
- **#11** — Fixed: recover removed (nothing in the function can panic)
- **#12** — Partly fixed: `Config.Validate()` added, but it is not called from `GetConfig()`. The `*` CORS+credentials check is not implemented
- **#13** — Documented only: `GetDefault` returns a pointer to a copy
- **#14** — Fixed: `Join` errors if the result escapes the base
- **#3** — Open: default flips deferred by decision (breaking change).
- Tests: `pkg/config/hardening_test.go`.
---
## Findings
### 1. `Manager` exposes a lock-free viper as a concurrent API (Critical, Locking)
`manager.go:10-13`, `manager.go:133-158`
```go
type Manager struct {
v *viper.Viper
}
...
func (m *Manager) Get(key string) interface{} { return m.v.Get(key) }
func (m *Manager) GetString(key string) string { return m.v.GetString(key) }
func (m *Manager) Set(key string, value interface{}) { m.v.Set(key, value) }
```
`viper.Viper` carries its configuration in plain maps (`override`, `config`, `defaults`, `aliases`,
…) and has **no mutex**. Verified against the module in use:
```
$ grep -n 'sync\.\|Lock()' $(go env GOMODCACHE)/github.com/spf13/viper@v1.21.0/viper.go
319: initWG := sync.WaitGroup{} # inside WatchConfig only
340: eventsWG := sync.WaitGroup{} # inside WatchConfig only
```
`Set` writes to `v.override`; `Get` reads across those maps. Because `GetConfigManager()` hands the
*same* `*Manager` to every caller, any code path that calls `Manager.Set` at runtime while another
goroutine reads config is a concurrent map read/write. Go's runtime detects this and issues
`fatal error: concurrent map read and map write` — which **`recover()` cannot catch**, so none of
the panic handlers elsewhere in the codebase will save the process.
This is latent-but-loaded: it needs one runtime `Set` to become a crash. `SetConfig`
(`manager.go:107-131`) performs eleven `m.v.Set` calls, so any dynamic reconfiguration triggers it.
**Recommendation:** add a `sync.RWMutex` to `Manager` and take it in every method that touches
`m.v` (including the `Option` functions at `manager.go:60-85`, which also mutate viper). Better:
load once into an immutable `*Config` at startup and pass that value around, keeping `Manager`
confined to startup.
### 2. Unsynchronised lazy singleton (High, Locking)
`manager.go:15-45`
```go
var configInstance *Manager
func GetConfigManager() *Manager {
if configInstance == nil {
configInstance = NewManager()
}
return configInstance
}
```
Classic check-then-act race: two concurrent first calls both see `nil`, both build a `Manager`,
and the two callers get *different* instances — so a `Set` through one is invisible through the
other. The unsynchronised pointer write races with the read.
Worse, `NewManager()` (`manager.go:27-45`) assigns `configInstance = &Manager{v: v}` at line 43 as
a **side effect**. So a caller who deliberately builds an isolated manager silently replaces the
global one, and `NewManagerWithOptions` (`manager.go:48-54`) publishes a half-configured manager to
the global *before* applying its options — another goroutine can observe the instance mid-mutation.
**Recommendation:** `sync.Once` for the singleton; remove the global assignment from `NewManager`.
### 3. Insecure-by-default configuration (High, Security)
`manager.go:203-247`:
```go
v.SetDefault("cors.allowed_origins", []string{"*"})
v.SetDefault("cors.allowed_headers", []string{"*"})
...
v.SetDefault("dbmanager.connections.default.user", "postgres")
v.SetDefault("dbmanager.connections.default.password", "")
v.SetDefault("dbmanager.connections.default.sslmode", "disable")
```
Each of these is inherited by an internet-facing layer:
- **`allowed_origins: ["*"]` + `allowed_headers: ["*"]`** — any origin may make cross-origin calls
with arbitrary headers. Whether this is exploitable depends on whether the CORS middleware also
sets `Access-Control-Allow-Credentials`; see `audit/pkg/middleware.audit.md` for that
determination. Even without credentials, wildcard origin plus wildcard headers defeats any
header-based CSRF defence and lets a malicious page read responses from a
network-position-authenticated deployment (IP allowlisted, mTLS-terminated, VPN).
- **`sslmode: disable`** — DB traffic unencrypted by default. Every row that crosses the wire,
including whatever the internet-facing handlers select, is plaintext on the network.
- **`user: postgres` with an empty password** — the default connection targets the PostgreSQL
superuser. Combined with the identifier-handling concerns in
`audit/pkg/common.audit.md` / `audit/pkg/restheadspec.audit.md`, running as superuser removes the
last line of defence (least-privilege) against a query-construction bug.
Because of finding 6, a deployment with a missing or misnamed config file runs on **all** of these
simultaneously and reports success.
**Recommendation:** default to `sslmode: require`, no default DB user/password (fail loudly if
unset), and `cors.allowed_origins: []` with wildcard requiring an explicit opt-in. Add a
`Config.Validate()` that refuses `allowed_origins: ["*"]` together with credentials.
### 4. `SaveConfig` writes secrets in plaintext at 0644 (High, Security)
`manager.go:160-166`
```go
func (m *Manager) SaveConfig(path string) error {
if err := m.v.WriteConfigAs(path); err != nil { ... }
}
```
`WriteConfigAs` serialises the **entire** merged configuration. That includes
`dbmanager.connections.*.password`, `cache.redis.password`, `event_broker.redis.password` and
`error_tracking.dsn` (a Sentry DSN is a credential).
Viper writes with `v.configPermissions`, which defaults to `0o644`
(`viper@v1.21.0/viper.go:198`). `SetConfigPermissions` is **never called anywhere in this repo**
(verified by grep), so the file is world-readable. Any local user or any other container sharing
the mount can read the DB superuser password.
**Recommendation:** call `v.SetConfigPermissions(0o600)` in `NewManager`; better, strip secret keys
before writing and document that secrets come from env/secret-manager only.
### 5. Current-working-directory config injection (Medium, Security)
`manager.go:32-36`
```go
v.AddConfigPath(".")
v.AddConfigPath("./config")
v.AddConfigPath("/etc/resolvespec")
v.AddConfigPath("$HOME/.resolvespec")
```
Viper searches these **in order** and takes the first hit, so `./config.yaml` wins over
`/etc/resolvespec/config.yaml`. For a daemon this is backwards: the CWD is the least trustworthy of
the four. If the process is ever started with its CWD in a shared or user-writable directory (a
tmp dir, a bind-mounted volume, `/` in some container setups), an attacker with local write
capability redirects the DB connection, disables TLS, or points `error_tracking.dsn` at their own
collector — turning finding 1 of `audit/pkg/errortracking.audit.md` into a full exfiltration path.
**Recommendation:** search `/etc/resolvespec` first, drop `"."` from the default list (keep it
available via `WithConfigPath`), and log the resolved path at startup (`v.ConfigFileUsed()`).
### 6. `Load()` is silent about a missing config file (Medium, Observability)
`manager.go:87-97`
```go
if err := m.v.ReadInConfig(); err != nil {
if _, ok := err.(viper.ConfigFileNotFoundError); !ok {
return fmt.Errorf("error reading config file: %w", err)
}
// Config file not found; will rely on defaults and env vars
}
return nil
```
The comment is the only trace. No log line, no returned indicator, no `ConfigFileUsed()` report.
A typo in the filename, a wrong working directory, or a container that forgot to mount the
ConfigMap is indistinguishable from a deliberate defaults-only run — and the defaults are the ones
in finding 3.
**Recommendation:** log at info level whether a file was used and which one; expose
`ConfigFileUsed()` on `Manager` so startup can print it.
### 7. `PathsConfig.Set` panics on a nil map (Medium, Panic handling)
`paths.go:38-40`
```go
func (pc PathsConfig) Set(name, path string) {
pc[name] = path
}
```
`PathsConfig` is `map[string]string` (`config.go:200`). `Get`, `GetOrDefault`, `Has` and `List` all
begin with `if pc == nil`. `Set` does not — and assignment to a nil map is
`panic: assignment to entry in nil map`.
`Config.Paths` is populated by `mapstructure`, which leaves the map nil when the `paths` key is
absent from the file. `setDefaults` does register `paths.data_dir` etc. (`manager.go:249-253`), so
the map is non-nil on the normal `GetConfig()` path — but a `Config` built in code
(`config.Config{}`) or produced by a partial unmarshal has a nil `Paths`, and `Set` on it panics.
Nothing in `pkg/` currently calls `Set` (verified by grep), so this is a latent API defect.
**Recommendation:** nil-guard consistently, or change the receiver to `*PathsConfig` so `Set` can
allocate.
### 8. `PathsConfig` has no synchronisation (Medium, Locking)
Same type: a bare map with a mutating `Set` and reading `Get`/`Has`/`List`/`EnsureDir`/`AbsPath`/
`Join`. If any consumer calls `Set` at runtime while request handlers resolve paths, that is a
concurrent map write — again the **unrecoverable** `fatal error` class, not a panic.
Currently unused outside the package, so severity is capped at Medium. If the intent is a runtime
path registry, it needs a mutex and an unexported map.
### 9. `GetIPs()` blocks on an uncontexted DNS lookup (Medium, Slowness)
`server.go:113-149`
```go
hostname, _ = os.Hostname()
...
addrs, err := net.LookupIP(hostname)
```
`net.LookupIP` has no context and no timeout override — it blocks for the resolver's own timeout,
which on a misconfigured or slow-resolver host is 5 s per attempt and up to ~15–20 s with retries
across `/etc/resolv.conf` entries. In a container whose hostname is not in DNS (the normal case)
this fails, but only *after* the resolver gives up.
There is no caller in `pkg/` today, so it is not on the request path yet. It is exported and
named like a utility, so the risk is that it lands on one.
Secondary correctness problem in the same function: the fallback branch (`server.go:139-147`)
appends `a.String()` for a `net.Addr` from `net.InterfaceAddrs()`, which renders as CIDR
(`192.168.1.5/24`), into the same comma-joined string that the primary branch fills with bare IPs.
Consumers get two formats from one field. That branch also never appends to `ipaddrlist`, so the
third return value is empty whenever the fallback is taken.
**Recommendation:** `net.DefaultResolver.LookupIPAddr(ctx, host)` with a short deadline; cache the
result; normalise the fallback to bare IPs via `net.Addr.(*net.IPNet).IP`.
### 10. `SetConfig` does dead work that can fail the call (Medium, Correctness)
`manager.go:107-131`
```go
configMap := make(map[string]interface{})
if err := m.v.Unmarshal(&configMap); err != nil {
return fmt.Errorf("failed to prepare config map: %w", err)
}
// configMap is never read again
m.v.Set("servers", cfg.Servers)
...
```
`configMap` is written and then never used. The comment says "Marshal the config to a map structure
that viper can use", but it unmarshals *viper's current state* into a throwaway map — it has
nothing to do with `cfg`. The only effect is that a decode error in the **existing** config makes
`SetConfig` fail for no reason. It also does a full reflective decode of the whole config tree on
every call.
Note also that `SetConfig` stores Go structs into viper via `Set`, and the eleven `Set` calls are
not atomic — a concurrent `GetConfig()` observes a torn config (new `servers`, old `cors`), on top
of finding 1's race.
**Recommendation:** delete the `configMap` block.
### 11. `GetIPs()` panic handling bypasses the logger (Low, Panic handling)
`server.go:114-118`
```go
defer func() {
if err := recover(); err != nil {
fmt.Println("Recovered in GetIPs", err)
}
}()
```
- Writes to stdout with `fmt.Println` rather than `logger.Error`/`logger.HandlePanic`, so the event
never reaches the error tracker and is invisible to structured log collection.
- No stack trace captured.
- The function's results are named (`hostname, ipList string, ipNetList []net.IP`) but the body
builds `iplist`/`ipaddrlist` **locals** and only assigns via the `return` statements. On a panic,
the deferred recover swallows it and the function returns the *zero* named values — `ipNetList`
is nil rather than the empty slice callers might expect. Silent empty success.
`pkg/config` is otherwise the only package outside `pkg/logger` that hand-rolls a recover instead
of using the shared helpers.
**Recommendation:** use `defer logger.CatchPanic("GetIPs")()`, or drop the recover — there is no
panicking operation in this function for it to catch.
### 12. No validation of numeric/limit settings (Low, Security)
`ServerInstanceConfig.Validate` (`server.go:37-68`) and `ServersConfig.Validate`
(`server.go:71-95`) are good — port range, mutually-exclusive TLS modes, cert/key pairing,
AutoTLS domains. But nothing validates:
- `middleware.rate_limit_rps` / `rate_limit_burst` — `0` disables rate limiting silently.
- `middleware.max_request_size` — `0` may mean unlimited depending on the middleware; see
`audit/pkg/middleware.audit.md`.
- `event_broker.worker_count` (default 10) — `0` means no consumers; see
`audit/pkg/eventbroker.audit.md` for whether that deadlocks publishers or drops events.
- `dbmanager.max_open_conns`, retry counts/delays — negative or zero values.
- `cors.allowed_origins: ["*"]` in combination with credentials.
There is also no top-level `Config.Validate()` that calls the section validators, so nothing
guarantees `ServersConfig.Validate` ever runs.
**Recommendation:** add `func (c *Config) Validate() error` that fans out to every section, and
call it from `GetConfig()`.
### 13. `GetDefault()` returns a pointer to a copy (Low, Correctness)
`server.go:98-110`
```go
instance, ok := sc.Instances[sc.DefaultServer]
...
return &instance, nil
```
`instance` is a copy of the map value. A caller that mutates through the returned pointer — which
the `*ServerInstanceConfig` receiver on `ApplyGlobalDefaults` (`server.go:12`) invites — changes
only the copy, and `sc.Instances` is unaffected. This is exactly the shape of bug where timeouts
appear to be applied but aren't.
**Recommendation:** make `Instances` a `map[string]*ServerInstanceConfig`, or return by value.
### 14. `PathsConfig.Join` does not confine to the base (Low, Security)
`paths.go:96-104`
```go
parts := append([]string{base}, elem...)
return filepath.Join(parts...), nil
```
`filepath.Join` calls `Clean`, which *resolves* `..` rather than rejecting it: `Join("data",
"../../etc/passwd")` returns `../etc/passwd`. Any consumer that passes a request-derived segment
gets directory traversal out of the configured base. No consumer does today, hence Low, but the
method's name promises confinement it does not provide.
**Recommendation:** after joining, verify `strings.HasPrefix(filepath.Clean(result), filepath.Clean(base)+string(os.PathSeparator))`, or use `os.Root`/`filepath.Localize` on the elements.
---
## What looks right
- `ServerInstanceConfig.Validate` / `ServersConfig.Validate` (`server.go:37-95`) are thorough:
port bounds, mutual exclusion of the three TLS modes, cert/key co-presence, AutoTLS domain
requirement, and a key-vs-`Name` consistency check on the instances map. This is the strongest
code in the package.
- `ApplyGlobalDefaults` (`server.go:12-32`) uses `*time.Duration` fields so "unset" is
distinguishable from "zero" — the right modelling choice, and it copies into a fresh local
before taking its address rather than aliasing the loop/parameter variable.
- `Load()` correctly distinguishes `ConfigFileNotFoundError` from real read errors instead of
treating every failure as fatal (the *silence* is the problem, not the branch).
- `SetEnvPrefix("RESOLVESPEC")` + `SetEnvKeyReplacer(".", "_")` + `AutomaticEnv`
(`manager.go:38-41`) is the correct trio for env overrides, and because every key has a
registered default, `AutomaticEnv` actually resolves nested keys — so secrets *can* be supplied
via env instead of the file. That's the mitigation for finding 4, and it should be documented as
the only supported way to pass secrets.
- The defaults table is comprehensive and one place — easy to review, which is how findings 3 and
12 were found.
- Test coverage is reasonable for a config package (608 LOC of tests against 1023 of source),
though it does not cover concurrency, `SaveConfig` permissions, or `PathsConfig.Set`.
## Suggested follow-up
1. Lock `Manager` or make config immutable after load (findings 1, 2). Until then, treat
`Manager.Set` as unsafe to call after startup and consider removing it from the public API.
2. Flip the insecure defaults and add `Config.Validate()` (findings 3, 12).
3. `SetConfigPermissions(0o600)` and secret-stripping in `SaveConfig` (finding 4).
4. Reorder the config search path and log the resolved file (findings 5, 6).
5. Delete the dead `Unmarshal` in `SetConfig` (finding 10).
+630
View File
@@ -0,0 +1,630 @@
# Audit: `pkg/dbmanager`
| | |
|---|---|
| **Package** | `github.com/bitechdev/ResolveSpec/pkg/dbmanager` (+ `providers/`) |
| **Files** | `config.go` (489), `connection.go` (722), `manager.go` (401), `metrics.go` (136), `errors.go` (82), `factory.go` (67), `providers/postgres.go` (231), `providers/postgres_listener.go` (401), `providers/sqlite.go` (216), `providers/mongodb.go` (214), `providers/mssql.go` (184), `providers/existing_db.go` (111), `providers/provider.go` (89); tests `factory_test.go` (369), `manager_test.go` (290), `providers/existing_db_test.go` (194), `providers/postgres_listener_example_test.go` (229) |
| **Audit date** | 2026-09-30 |
| **Axes** | thread locking/waiting, slowness, security, panic handling & logging |
| **Threat model** | hostile internet client; request bodies, headers, query params, schema/table/column names all attacker-controlled |
| **Depth** | deep (hot package; every request's DB handle comes from here). Several findings were checked with a throw-away probe test against SQLite, and the probe was deleted afterwards |
## Summary
`pkg/dbmanager` owns every database pool in the process. It wraps a
`*sql.DB` (or a `mongo.Client`) in a `sqlConnection` and hands out lazily-built
`*bun.DB`, `*gorm.DB`, raw `*sql.DB` and `common.Database` adapters over it. A
background health checker pings each connection every 15 s, and it can
**reconnect**, which closes the pool and opens a new one.
This audit was started to answer one question: **"why does a database
connection that has been idle for a while become unusable?"** Several defects
in this package combine to give exactly that symptom. They are findings 1–5,
and the [Idle-connection failure chain](#idle-connection-failure-chain) section
below puts them together.
The root design problem is that **`Reconnect` destroys the shared `*sql.DB`**.
`*sql.DB` is already a self-healing pool: it throws away bad connections and
dials new ones. So "reconnecting" a pool is almost never needed, and here it
has a large blast radius. Every `*bun.DB`, `*gorm.DB` and `*sql.DB` handed out
before the reconnect now points at a closed pool, and it stays closed. Only the
`common.Database` adapters carry a factory that can re-fetch a handle, and even
they only use it on a subset of code paths (see `common.audit.md` finding 5).
Those adapter factories also *trigger* `Reconnect` themselves, so one stale
handle closes the pool for everyone else. `Reconnect` isn't atomic, so
concurrent callers turn this into a storm.
The other major theme is **missing client-side deadlines**. `QueryTimeout` is
only ever sent to the server as `statement_timeout`, which does nothing when
the TCP peer has vanished. No `context.WithTimeout` is applied to request
queries, and pgx's dialer sets no `TCP_USER_TIMEOUT`. So the first query on a
pooled connection whose peer silently disappeared (NAT/firewall idle drop,
failover, a pgbouncer restart) can block for minutes. One Close path does this
while holding the connection's write lock, which stalls every request.
## Findings
| # | Severity | Axis | Finding | Status |
|---|---|---|---|---|
| 1 | **Critical** | locking / availability | `Reconnect` closes the shared `*sql.DB`, so every `*bun.DB` / `*gorm.DB` / `*sql.DB` handed out earlier is permanently dead ("sql: database is closed") | Fixed |
| 2 | **High** | locking | Adapter reconnect factories call `Reconnect` on the *shared* connection, and `Reconnect` is not atomic, so one stale handle starts a reconnect storm that repeatedly closes the pool under in-flight requests | Fixed |
| 3 | **High** | slowness / locking | `sqlConnection.HealthCheck` holds the write lock across a network ping for up to 5 s; every `Bun()`/`GORM()`/`Native()`/`Database()`/`Stats()` call blocks for that time | Fixed |
| 4 | **High** | slowness | No client-side query deadline and no `TCP_USER_TIMEOUT`: a query on a silently-dead idle socket blocks for minutes (up to about 15 min); `QueryTimeout` is server-side only, and is forced to at least 2 min | Fixed |
| 5 | **High** | locking / slowness | `PostgresListener.Close` runs `UNLISTEN` with `context.Background()` while `sqlConnection.mu` (write), `PostgresProvider.mu` and `listener.mu` are all held; on a dead socket this freezes every request for minutes | Fixed |
| 6 | **High** | locking / leak | `PostgresListener.Connect` starts a new goroutine pair on every (re)connect; the old pair keeps running, so two loops call `WaitForNotification` on one `pgx.Conn` concurrently, which triggers more reconnects | Fixed |
| 7 | **High** | panic handling | `Connect → Close → Connect → Close` panics with "close of closed channel"; after the first cycle the health checker also exits immediately and silently | Fixed |
| 8 | **Medium** | availability | SQLite: `:memory:` with a 25-connection pool gives every connection its own empty database, and `ConnMaxIdleTime` then silently discards data; `busy_timeout` / WAL pragmas are applied to only one pooled connection | Fixed |
| 9 | **Medium** | availability | Partial failure in `sqlConnection.Close` leaves `connected=true` over a closed pool; partial failure in `Manager.Connect` leaks the connections already opened | Fixed |
| 10 | **Medium** | security | DSN builders concatenate unescaped credentials (postgres key=value, mssql/mongo URLs); `sslmode` defaults to `disable` | Fixed |
| 11 | **Medium** | config | Several config knobs are ignored or impossible to turn off: `EnableAutoReconnect`, `HealthCheckInterval`, `RetryAttempts`/`RetryDelay`/`RetryMaxDelay`, SQLite `_timeout`, and `statement_timeout` when a DSN is given | Fixed |
| 12 | **Medium** | locking | `Manager.Connect` holds `m.mu` across every network dial (up to 3 retries × `ConnectTimeout` per connection) | Fixed |
| 13 | **Low** | observability | `PublishMetrics` / `RecordReconnectAttempt` are never called, so all dbmanager metrics are permanently zero; `*_total` metrics are gauges | Fixed |
| 14 | **Low** | correctness | `Bun()`/`GORM()` do not check `connected`; `getNativeAdapter` uses `PgSQLAdapter` for SQLite and MSSQL; `ExistingDBProvider` applies no pool settings and closes the caller's DB | Fixed (partly, see notes) |
| 15 | **Low** | logging | `Close` / `performHealthCheck` pass key-value pairs to the printf-style logger, which produces `%!(EXTRA ...)` output; `ResetInstance` discards the close error | Fixed |
## Remediation status
Implemented 2026-09-30. `go build ./...` and `go test -race ./pkg/dbmanager/...`
pass. The Postgres behaviour was also verified against a live server (tests are
skipped unless `PG_LIVE=1` / `PG_RESTART_DIR` is set).
**Design decisions taken**
- No automatic reconnect. Adapter factories and the health checker never close
the pool; they only re-fetch the current handle. `*sql.DB` replaces bad
connections itself. `EnableAutoReconnect` is deprecated and ignored.
- `Reconnect` is atomic (one critical section) and operator-only. On PostgreSQL
it goes through a custom `driver.Connector` (`providers/pgconnector.go`): it
bumps a generation, stale pooled connections are discarded, and the `*sql.DB`
is never closed, so held Bun/GORM/`*sql.DB` handles keep working. Other
providers still close and reopen.
- Client-side deadlines are applied at the driver level rather than in the
adapters (a `context.WithTimeout` around a query is cancelled before the
caller has read the rows).
**Per finding**
1. Fixed. Postgres refresh keeps the pool; explicit `Reconnect` on other
providers still invalidates handles (documented in the README).
2. Fixed. Adapter factories no longer call `Reconnect`; `Reconnect` is a single
critical section under `lifecycleMu` + `mu`.
3. Fixed. The ping runs without `mu`; `lifecycleMu` (read) only keeps
`Close`/`Reconnect` from tearing the provider down mid-ping. Same for Mongo.
4. Fixed. TCP keepalive and `TCP_USER_TIMEOUT` (30 s, Linux) via `DialFunc`;
the reuse-time liveness ping is capped at 5 s; `statement_timeout` is set as
a runtime parameter so it also applies to a supplied DSN; the 2-minute floor
on `QueryTimeout` is removed. `SetConnMaxIdleTime` tuning remains a
configuration matter (documented in the README).
5. Fixed. Listener `Close` sends no `UNLISTEN`, closes with a 2 s bound, and
holds no lock across network I/O.
6. Fixed. Background goroutines start once (`sync.Once`); reconnect dials a
replacement, re-`LISTEN`s, then swaps it in; sleeps honour `ctx.Done()`.
Additionally, all use of the single `pgx.Conn` is serialised (`connMu`, 500 ms
notification poll), fixing "conn busy" from `Listen`/`Unlisten`/`Notify`, and
old connections are closed under `connMu` (a race found by the live test).
7. Fixed. Stop channel is created per start, guarded by `healthMu`; `Close` is
idempotent; `Connect` is idempotent.
8. Fixed. `:memory:` is pinned to one connection with no idle/lifetime limits;
`busy_timeout`/WAL are `_pragma` DSN parameters; `_timeout` and the dead
reconnect code are removed.
9. Fixed. `Close` always marks disconnected and returns joined errors;
`PostgresProvider.Close` closes the pool even if the listener fails;
`Manager.Connect` closes connections it opened when a later one fails.
10. Fixed. Postgres, MSSQL and Mongo DSNs are built as escaped URLs; default
`sslmode` is now `prefer` (was `disable`).
11. Fixed. Retry settings reach every provider; a negative
`HealthCheckInterval` disables the health checker; `EnableAutoReconnect`
deprecated; `statement_timeout` applies with a supplied DSN.
12. Fixed. `Manager.Connect` dials outside `m.mu` and publishes results under it.
13. Fixed. `PublishMetrics` runs on each health-check tick, `Reconnect` records
`RecordReconnectAttempt`, and the wait/closed metrics are true counters
(delta-tracked).
14. Partly fixed. `Bun()`/`GORM()` check `connected`; Mongo no longer maps
`MaxIdleConns` to `MinPoolSize`. `ExistingDBProvider`: `Close` is now a no-op
that logs a warning (the caller owns the `*sql.DB`; the connection's `Close`
also skips `bun.DB.Close`), and `Reconnect` only pings. Pool settings are
still not applied to a caller-owned pool. The `getNativeAdapter` claim was
stale: the adapter already receives the driver name; the three duplicate
cases were merged. Mongo `Stats()` is still empty.
15. Fixed. Printf-style logger calls corrected; `ResetInstance` logs the close
error. Unscrubbed driver errors in Sentry (X8) are not addressed here.
**Behaviour changes**
- Removed tests that closed the pool from outside and expected an adapter to
swap in a new one (three adapter tests, and the health-check reconnect test,
now asserting it never reconnects).
- `sslmode` default `prefer`; `NewConnectionFromDB` connections are no longer
closed by the manager.
**Regression tests added:** `lifecycle_test.go` (double Connect/Close cycle,
idempotent Connect, concurrent Reconnect, adapter factory leaves pool open,
accessors not blocked by health check, Close marks disconnected, existing-DB
Reconnect/Close leave the caller's pool open), `config_dsn_test.go`,
`providers/pgconnector_test.go`, `pg_live_test.go` (refresh keeps handles,
listener Listen/Notify) and `restart_live_test.go` (server crash and restart).
---
## Idle-connection failure chain
This is how findings 1–5 combine into "the connection sat idle and then could
not be used":
1. The app is idle. A NAT, firewall, load balancer or pgbouncer silently drops
the idle TCP flows. No FIN or RST reaches the process.
2. The next request takes a pooled connection. pgx's `ResetSession` pings it
because it has been idle for more than 1 s, and that ping uses the request
ctx, **which has no deadline** (finding 4). The write goes into the kernel
buffer and the read blocks until TCP retransmission gives up, which can
take minutes.
Meanwhile the health checker's 5 s ping times out and holds `c.mu`
**exclusively** for the whole time (finding 3), so every request trying to
get a handle queues behind it.
3. Eventually something returns "sql: database is closed" or
`ErrConnectionClosed`. That can be an adapter that hit a closed pool, or a
partial `Close` (finding 9). An adapter's `dbFactory` or the health checker
then calls `Reconnect` (finding 2).
4. `Reconnect` closes the `*sql.DB` (finding 1). If the Postgres listener has
subscriptions, `Close` first sends `UNLISTEN` on its own dead socket with no
deadline, still holding the write lock (finding 5), which freezes the
process again.
5. When the reconnect completes, every handle captured before it is
permanently broken. That includes the `*gorm.DB` given to
`resolvespec.NewHandlerWithGORM` in `cmd/testserver/main.go:142,56`, any
`*bun.DB` passed to `NewHandlerWithBun`, and every Bun `NewSelect`/`NewInsert`
path. **From this point on, every request that goes through those handles
fails until the process is restarted.** Concurrent failures run their own
`Reconnect`s, and each one closes the pool the previous one just opened
(finding 2).
### Fix order for this symptom
1. **Stop closing the pool to recover from connection errors.** Remove
`WithDBFactory(c.reopen*ForAdapter)` → `Reconnect`, and remove the
health-check → `Reconnect` path for SQL providers. `*sql.DB` already discards
bad connections (`driver.ErrBadConn`, `ResetSession`,
`SetConnMaxIdleTime`/`SetConnMaxLifetime`). Keep `Reconnect` for explicit
operator use only, and make it atomic (finding 2).
2. Give every request a deadline. Wrap the request ctx in
`context.WithTimeout(ctx, QueryTimeout)` in the adapters, or at the handler
boundary.
3. Set `SetConnMaxIdleTime` **below** the shortest idle timeout of any
middlebox (typically 60–240 s for cloud NATs and LBs) so idle connections
are recycled before they can be dropped silently. Also set TCP keepalive and
`TCP_USER_TIMEOUT` through a custom `pgconn.Config.DialFunc`.
4. Ping without the write lock (finding 3), and give the listener's `Close`
bounded ctxs (finding 5).
---
### 1. Critical — `Reconnect` kills every previously issued handle
`connection.go:129-160` (`Close`) and `connection.go:187-192` (`Reconnect`):
```go
func (c *sqlConnection) Close() error {
c.mu.Lock()
...
if c.bunDB != nil {
if err := c.bunDB.Close(); err != nil { // closes the shared *sql.DB
...
if err := c.provider.Close(); err != nil { // closes it again (idempotent)
...
c.nativeDB = nil
c.bunDB = nil
c.gormDB = nil
c.bunAdapter = nil
...
}
func (c *sqlConnection) Reconnect(ctx context.Context) error {
if err := c.Close(); err != nil {
return err
}
return c.Connect(ctx)
}
```
`Bun()`, `GORM()` and `Native()` return the handle itself, and callers keep
it: every spec package has a `NewHandlerWithGORM(*gorm.DB)` /
`NewHandlerWithBun(*bun.DB)` constructor, and `cmd/testserver/main.go:142` does
exactly this. After `Reconnect`, the cached fields are nilled, a new pool is
built, and the handles the callers hold point at a `*sql.DB` whose `closed`
flag is set forever.
Verified with a probe: I obtained `conn.GORM()`, called `conn.Reconnect(ctx)`,
then ran a query through the old handle. It returned
`sql: database is closed`, and a fresh `conn.GORM()` worked.
The comment in `manager.go:371-374` shows the authors already knew about this
("forcing Close()+Connect() here invalidates any cached ORM wrappers and callers
that still hold the old handle"). Their mitigation was to narrow *when* the
health checker reconnects. But the adapters' own `dbFactory` still reconnects
unconditionally (finding 2).
**Failure scenario.** Any event that triggers a reconnect turns every
long-lived handler into a permanent 500 generator: a single adapter query hitting
"database is closed", or a health check returning `ErrConnectionClosed`. The
process does not recover without a restart. The same thing happens after a
normal `Manager.Close()` + `Connect()` in tests or hot-reload code.
**Recommendation.** Treat the `*sql.DB` as immortal for the life of the
`sqlConnection`. Don't close it to "reconnect": `database/sql` already replaces
broken connections. If a real re-dial is ever needed (for example after
changing credentials), build the new pool, atomically swap it in, and close the
old one only after a grace period. Give the handles returned by
`Bun()`/`GORM()`/`Native()` stable identity; one way is a `driver.Connector`
that indirects to the current pool.
---
### 2. High — Adapter-triggered, non-atomic `Reconnect` causes a reconnect storm
`connection.go:362-397` and `connection.go:431/474/517-525`:
```go
func (c *sqlConnection) reconnectForAdapter() error {
...
return c.Reconnect(ctx) // Close() then Connect(): two separate lock scopes
}
...
WithDBFactory(c.reopenBunForAdapter).
```
The adapters (`pkg/common/adapters/database/bun.go:131`, `gorm.go`,
`pgsql.go`) call `dbFactory` whenever an operation returns an error that
matches `"sql: database is closed"`. So:
- **One stale handle closes the pool for everyone.** If an adapter holds a
`*sql.DB` from before a previous reconnect, its first query fails with
"database is closed". Its factory then calls `c.Reconnect`, which closes the
*current, healthy* pool that every other adapter and request is using right
now.
- **`Reconnect` isn't atomic.** `Close` and `Connect` each take `c.mu`
separately. Under N concurrent failures, one goroutine closes and reconnects
while the others either close the brand-new pool again or fail with
`already connected`. The probe used 20 concurrent `Reconnect`s: 9 returned
"already connected", and every successful reconnect closed the pool the
previous winner had just handed to its adapter. Each of those adapters then
sees "database is closed" on its next query, and the cycle continues.
**Failure scenario.** A burst of traffic arrives just after a reconnect. Each
in-flight request whose adapter still holds the old pool triggers another
`Reconnect`, and each of those closes the pool that the previous request
reopened. The service flaps until traffic stops.
**Recommendation.** Remove the adapter → `Reconnect` path (see finding 1). If
it is kept, make `Reconnect` a single critical section, and add a generation
counter: a caller that saw generation N only reconnects if the current
generation is still N; otherwise it just re-fetches the handle.
---
### 3. High — Health check holds the write lock across a network ping
`connection.go:163-185`:
```go
func (c *sqlConnection) HealthCheck(ctx context.Context) error {
c.mu.Lock() // exclusive
defer c.mu.Unlock()
...
if err := c.provider.HealthCheck(ctx); err != nil { // PingContext, 5 s timeout
```
Every handle accessor takes `c.mu.RLock()` first (`connection.go:199, 238, 271,
308, 335, 403, 441, 484`). While the health checker (every 15 s, `manager.go:348`)
is pinging, **every request that needs a DB handle waits**. On a healthy
network this is a few ms. On a dead idle socket it's the full 5 s ping timeout
(`providers/postgres.go:155`, inside a 10 s outer ctx).
Verified with a probe: while `c.mu` was held, `conn.Bun()` blocked for the whole
hold (200 ms in the test).
**Failure scenario.** A network blip or a silently dropped idle connection
makes the ping hang. Every 15 s the whole API pauses for up to 5 s. This fits
reports of "idle, then slow or unusable".
**Recommendation.** Snapshot `provider` under `RLock`, release the lock, ping,
then take the lock only to write `healthCheckStatus` / `lastHealthCheck`. Better
still, keep the status in an `atomic.Value`.
---
### 4. High — No client-side query deadline; `QueryTimeout` is server-side only and floored at 2 min
`config.go:223-228`:
```go
if cc.QueryTimeout == 0 {
cc.QueryTimeout = 2 * time.Minute
} else if cc.QueryTimeout < 2*time.Minute {
cc.QueryTimeout = 2 * time.Minute
}
```
`config.go:331-335` turns this into `statement_timeout=<ms>` in the Postgres DSN,
and it only does that when the DSN is *built*. A user-supplied `DSN` gets no
timeout at all. Nothing anywhere in the request path wraps ctx in a deadline.
`pkg/config`'s `query_timeout: 30s` default is silently raised to 2 min.
`statement_timeout` is enforced by the **server**, so it only helps if the
server is reachable. On a silently dropped connection:
- pgconn's default dialer is `&net.Dialer{}`: Go's default keepalive (15 s idle,
15 s interval, 9 probes) and **no `TCP_USER_TIMEOUT`**.
- Once a query has been written, there is unacknowledged data, so keepalive does
not apply. The socket then waits for TCP retransmission to give up
(`tcp_retries2`), which takes about 15 min on Linux defaults.
- `database/sql` calls pgx's `ResetSession`, which pings a connection that has
been idle for more than 1 s. That ping uses the **request ctx**, so with no
deadline it blocks just as long.
**Failure scenario.** An idle period longer than the NAT or LB idle timeout
causes the next request to hang for minutes rather than failing fast and being
retried on a fresh connection. With `MaxOpenConns` = 25, 25 such requests
exhaust the pool and every later request blocks on `db.conn()`.
**Recommendation.**
- Apply `context.WithTimeout(ctx, QueryTimeout)` in the adapters, or in a
handler middleware.
- Remove the 2-minute floor, and honour the configured value.
- Set `SetConnMaxIdleTime` below the middlebox idle timeout.
- Configure `pgconn.Config.DialFunc` with a `net.Dialer` that has `KeepAlive`
set and a `Control` func setting `TCP_USER_TIMEOUT` (for example 30 s).
- Apply `statement_timeout` through `RuntimeParams` so it also works with a
supplied DSN.
---
### 5. High — Listener `Close` does unbounded network I/O under three locks
`providers/postgres_listener.go:216-244`, reached from
`providers/postgres.go:116-126`, which is reached from `connection.go:147`:
```go
// sqlConnection.Close holds c.mu (write)
// PostgresProvider.Close holds p.mu
// PostgresListener.Close holds l.mu:
for channel := range l.channels {
_, _ = l.conn.Exec(context.Background(), fmt.Sprintf("UNLISTEN %s", ...))
}
err := l.conn.Close(context.Background())
```
If the listener's socket is dead, and it usually is in the situation that
triggers a reconnect, each `UNLISTEN` waits for a reply that never comes. This
is the same unbounded wait as in finding 4, and `c.mu` is held **for writing**
the whole time. Every request blocks. `bunDB` has already been closed at this
point, so there is no fallback either.
Also, if `listener.Close` returns an error, `PostgresProvider.Close` returns
early. `sqlConnection.Close` then returns with `connected=true` over a closed
pool (finding 9).
**Failure scenario.** An app with any `LISTEN` subscription hits a network
partition. The health checker or an adapter calls `Reconnect`, and the process
stops serving database requests for as long as the kernel takes to kill the
socket.
**Recommendation.** Skip `UNLISTEN` entirely, because closing the connection
drops all subscriptions server-side. Close with `context.WithTimeout(…, 2*time.Second)`.
Don't do network I/O while holding `l.mu`, and don't close the listener
inside `sqlConnection.Close`'s write lock.
---
### 6. High — Listener leaks a goroutine pair per reconnect, and they race on one `pgx.Conn`
`providers/postgres_listener.go:48-120` (Connect), `257-324` (handleNotifications),
`326-370` (handleReconnection).
`Connect()` ends by starting `go l.handleNotifications()` and
`go l.handleReconnection()`. `handleReconnection` responds to a reconnect
signal by calling `l.Connect(ctx)`, which starts **another** pair. The old pair
keeps running on the same `l.ctx`. After N reconnects there are N+1
notification loops. Each one snapshots `l.conn` and calls
`conn.WaitForNotification`. `pgx.Conn` is **not** safe for concurrent use, so
the second caller gets a "conn busy" error. That error isn't a timeout, so it
sends another reconnect signal, which adds another pair.
`handleReconnection` also waits with `time.Sleep(5 * time.Second)` instead of
selecting on `l.ctx.Done()`, so `Close` can't interrupt it. And `Listen` runs
`l.conn.Exec(LISTEN …)` while holding `l.mu`, which blocks `handleReconnection`
for as long as that Exec takes.
Once the parent `PostgresProvider` is closed (for example by any `Reconnect`,
finding 1), subscribers holding the old `*PostgresListener` get
"listener is closed" forever. Nothing re-subscribes them on the new provider.
**Failure scenario.** A flaky network causes a few listener reconnects. The
goroutine count grows without bound, notifications are delivered twice or
dropped, and CPU rises because of the busy/reconnect spiral.
**Recommendation.** Start the goroutines once, in the constructor or the first
`Connect`. Have `handleReconnection` dial a new conn without calling the public
`Connect`. Guard `WaitForNotification` so only one loop owns the conn. Replace
`time.Sleep` with `select { case <-time.After(d): case <-l.ctx.Done(): }`.
---
### 7. High — Second `Close` panics; health checker silently dead after first cycle
`manager.go:119, 313-345`:
```go
stopChan: make(chan struct{}), // created once, in the constructor
...
func (m *connectionManager) stopHealthChecker() {
if m.healthTicker != nil {
m.healthTicker.Stop()
close(m.stopChan) // never recreated
m.wg.Wait()
m.healthTicker = nil
}
}
```
After `Connect → Close`, `stopChan` is closed. A second `Connect` calls
`startHealthChecker`, which creates a new ticker and goroutine. That goroutine's
`select` sees the closed `stopChan` right away and **exits**, so health
checking is silently off. A second `Close` finds `healthTicker != nil` and
calls `close(m.stopChan)` again, which **panics**: `close of closed channel`.
`startHealthChecker` and `stopHealthChecker` also read and write `healthTicker`
without `m.mu` held (`Close` calls `stopHealthChecker` before locking), so a
concurrent `Connect`/`Close` pair is a data race.
Calling `Connect` twice without `Close` also leaks: `m.connections[name] = conn`
overwrites the previous connection without closing it.
**Failure scenario.** Anything that cycles the manager can crash the process
during shutdown: graceful restart, config hot-reload, or test suites using
`ResetInstance`.
**Recommendation.** Create `stopChan` in `startHealthChecker`. Guard both
functions with `m.mu`, or a dedicated mutex. Make `Connect` idempotent, or have
it close existing connections first.
---
### 8. Medium — SQLite: in-memory data loss and per-connection pragmas
`providers/sqlite.go:54-90`, `config.go:140-141, 202-204`:
- `ManagerConfig.ApplyDefaults` always gives `MaxOpenConns` a value (25), so the
"SQLite works best with MaxOpenConns=1" branch at `sqlite.go:60` never runs.
The probe reported `MaxOpenConnections=25`.
- With `:memory:` (the documented test setup), each pooled connection opens its
**own** private database. The probe created a table on one connection, and a
second connection reported `no such table: t`. `ConnMaxIdleTime` (default
5 min) then closes idle connections and their data with them.
- `PRAGMA journal_mode=WAL` and `PRAGMA busy_timeout` are `Exec`'d once on
whichever pooled connection runs them. `busy_timeout` is per-connection, so
the other 24 get `database is locked` immediately under write contention.
- `BuildDSN` adds `?_timeout=<ms>` (`config.go:347-351`), but
`glebarez/go-sqlite` only recognises `_pragma`, `_txlock` and `_time_format`,
so this parameter is silently ignored.
- `SQLiteProvider.reconnectDB` (`sqlite.go:165`) needs a `dbFactory` that
nothing ever sets, so it is dead code.
**Recommendation.** For SQLite, force `MaxOpenConns=1` for `:memory:` (or use
`file::memory:?cache=shared`), and never set an idle timeout there. Pass the
pragmas in the DSN (`_pragma=busy_timeout(5000)&_pragma=journal_mode(WAL)`) so
every connection gets them.
---
### 9. Medium — Partial-failure states in `Close` and `Connect`
- `connection.go:137-149`: if `bunDB.Close()` or `provider.Close()` fails, for
example because the listener's Close failed (finding 5), `Close` returns
early with `connected = true` and the pool already closed. Every accessor then
returns a handle to a closed pool until someone calls `Close` again.
- `manager.go:197-231`: if connection *k* of *n* fails to connect, `Connect`
returns an error. Connections 1…k-1 stay open but are never stored in
`m.connections`, so `Close` can't reach them and they leak.
**Recommendation.** In `Close`, mark the connection disconnected and nil the
fields regardless of errors, and return a joined error. In `Connect`, close any
connections opened so far when a later one fails.
---
### 10. Medium — DSN builders don't escape credentials; TLS off by default
`config.go` `buildPostgresDSN` / `buildMSSQLDSN` / `buildMongoDSN` use
`fmt.Sprintf` with raw `User`/`Password`/`Database` values:
- Postgres key=value format: a password containing a space or `'` breaks
parsing. A password like `x sslmode=disable` *overrides earlier parameters*.
- MSSQL and Mongo URLs: `@`, `:`, `/`, `?` or `&` in the password corrupt the
URL. They need `url.QueryEscape` / `url.UserPassword`.
- `sslmode` defaults to `disable` (`config.go:322-325`); see
`_CROSS-CUTTING.audit.md` X6.
These values come from config, not from clients, so this isn't directly
exploitable by the threat model. It is a correctness and hardening problem,
and it becomes a security problem wherever DSN parts come from a tenant or
operator UI.
**Recommendation.** Build the Postgres DSN as a URL with `url.URL{User: url.UserPassword(...)}`,
or quote key=value values properly. Default `sslmode` to `prefer` or `require`.
---
### 11. Medium — Config knobs that are ignored or cannot be disabled
- `config.go:161-168`: `HealthCheckInterval == 0` and
`EnableAutoReconnect == false` are both treated as "unset" and replaced with
the defaults (15 s, `true`). **Auto-reconnect, the trigger for findings 1–2,
cannot be switched off from config.**
- `RetryAttempts`, `RetryDelay` and `RetryMaxDelay` are defaulted and copied,
but no provider reads them. Every provider hardcodes `retryAttempts := 3`
and `retryDelay := 1 * time.Second`.
- `statement_timeout` is only added when the DSN is built (finding 4), and
SQLite `_timeout` is ignored by the driver (finding 8).
**Recommendation.** Use `*bool` / `*time.Duration`, or an explicit
`Disable…` flag, for the values that can legitimately be zero or false. Wire
the retry settings into the providers, or delete them.
---
### 12. Medium — `Manager.Connect` holds the manager lock across network dials
`manager.go:197-231` holds `m.mu` (write) while dialing every configured
connection, each with up to 3 attempts, backoff, and `ConnectTimeout`.
`GetConnection`, `HealthCheck`, `Stats` and the health checker all wait
behind it. That's harmless at startup, but it serialises the whole manager if
`Connect` is ever called at runtime (hot-reload, lazy init).
**Recommendation.** Dial outside the lock, then lock only to publish the
results into `m.connections`.
---
### 13. Low — dbmanager metrics are never published
`metrics.go` defines Prometheus collectors plus `PublishMetrics` and
`RecordReconnectAttempt`. A grep over the repository finds **no callers** of
either. The connection-pool gauges (open, in-use, idle, wait count) are exactly
what would have shown the idle-connection problem, and they are always zero.
The `*_total` names are registered as gauges, not counters.
**Recommendation.** Call `PublishMetrics` from the health-check tick, call
`RecordReconnectAttempt` from `Reconnect`, and make the totals counters.
---
### 14. Low — Assorted correctness issues
- `Native()` checks `c.connected` (`connection.go:214`); `Bun()` and `GORM()`
don't. After a partial `Close` they can build ORM wrappers over a nil or
closed DB.
- `getNativeAdapter` (`connection.go:500-525`) wraps SQLite and MSSQL in
`PgSQLAdapter`, which quotes and builds SQL in Postgres dialect.
- `ExistingDBProvider` (`NewConnectionFromDB`) applies no pool settings and no
idle or lifetime limits, and its `Close` closes the caller's `*sql.DB`.
- `MongoProvider` uses `MaxIdleConns` as `MinPoolSize`, and `Stats()` returns an
empty struct.
---
### 15. Low — Logging defects
- `manager.go:247, 367-369, 378-380` call `logger.Error("…", "name", name, "error", err)`.
`pkg/logger` is printf-style, so these print `%!(EXTRA string=name, …)`, and
the error text is buried in exactly the log lines needed during an outage.
- `ResetInstance` discards the error from `Close`.
- Connection errors wrap driver errors that can include the DSN host and user.
Together with `_CROSS-CUTTING.audit.md` X8, they reach Sentry unscrubbed.
---
## Test coverage
`manager_test.go` and `factory_test.go` cover construction and config defaults.
Nothing tests `Reconnect` while handles are held, concurrent `Reconnect`, a
`Connect`/`Close` cycle run twice, health-check lock hold time, or listener
reconnection. Each of findings 1, 2, 3, 6 and 7 can be reproduced with a short
SQLite-backed test (the probes used for this audit took about 20 lines each).
Add them as regression tests when the fixes land, and run them with `-race`
(`_CROSS-CUTTING.audit.md` X1).
+220
View File
@@ -0,0 +1,220 @@
# Audit — `pkg/errortracking`
- **Date:** 2026-09-29
- **Scope:** `pkg/errortracking/{interfaces,noop,sentry,factory}.go` (260 LOC, 4 source files + 1 test file, 67 LOC)
- **Axes:** thread locking/waiting · slowness · security · panic handling & logging
- **Threat model:** hostile internet client; error messages and `extra` maps may contain attacker-shaped content.
## Summary
Small, clean abstraction: a `Provider` interface, a no-op implementation, a Sentry implementation,
and a config-driven factory. The concurrency story is fine — `sentry.Hub` is internally
mutex-guarded and the provider holds no mutable state of its own. The real exposure is **what
this package sends out of the trust boundary**: it is the egress point for every `Warn`/`Error`
in the codebase (see `audit/pkg/logger.audit.md` findings 2 and 3) and it applies **no scrubbing
whatsoever**.
| # | Severity | Axis | Finding |
|---|----------|------|---------|
| 1 | **High** | Security | No `BeforeSend` scrubber — messages, stack traces and `extra` leave the trust boundary verbatim |
| 2 | Medium | Security | `sentry.Init` mutates process-global state; `NewSentryProvider` can be called repeatedly and silently replaces the global client |
| 3 | Medium | Slowness | `Flush(timeout int)` is second-granularity only; combined with `Close()` gives up to 7 s of shutdown stall |
| 4 | Medium | Slowness | `CapturePanic` stringifies the whole stack trace into an `extra` field on every panic |
| 5 | Low | Security | `AttachStacktrace: true` is hardcoded — source paths and function names of the deployment leak to the SaaS |
| 6 | Low | Correctness | `CaptureError` produces an `Exception` with a nil `Stacktrace` for plain `errors.New` values |
| 7 | Low | Correctness | Config-provided `SampleRate == 0` silently means "send everything", not "send nothing" |
| 8 | Low | Architecture | `factory.go` imports `pkg/config`, coupling the lowest-level package to the config layer |
---
## Findings
### 1. No scrubbing before egress (High, Security)
`sentry.go:29-42`
```go
err := sentry.Init(sentry.ClientOptions{
Dsn: config.DSN,
Environment: config.Environment,
Release: config.Release,
Debug: config.Debug,
AttachStacktrace: true,
SampleRate: config.SampleRate,
TracesSampleRate: config.TracesSampleRate,
})
```
`BeforeSend` is not set. Neither is `BeforeSendTransaction`. Nothing in `CaptureError`
(`sentry.go:46`), `CaptureMessage` (`sentry.go:75`) or `CapturePanic` (`sentry.go:97`) inspects or
redacts its inputs; all three copy straight into `event.Message` / `event.Exception.Value` /
`event.Contexts["extra"]` and hand it to `hub.CaptureEvent`.
Because `pkg/logger.Error`/`Warn` forward every formatted message here unconditionally, the set of
things that can reach Sentry is "every error string produced anywhere in ResolveSpec". In this
codebase that includes driver errors (which embed DSNs and sometimes credentials on connect
failure), SQL fragments with bound values, and identifiers taken from request headers.
Under the hostile-client threat model this is an **attacker-reachable exfiltration channel**: shape
an input that lands in an error message, and its content is written to a third-party system
outside the operator's control.
**Recommendation:** set `BeforeSend` to run a redaction pass over `Message`,
`Exception[].Value` and `Contexts` — at minimum strip `password=`, `://user:pass@`, `Bearer `,
and anything matching the configured DSN patterns. Consider an `extra`-key allowlist rather than
passing the caller's map through (`sentry.go:70`, `92`, `114-121`).
### 2. `sentry.Init` mutates process-global state (Medium, Security/Correctness)
`sentry.go:29` calls the package-level `sentry.Init`, which installs a global client, and
`sentry.go:40` then captures `sentry.CurrentHub()`. Consequences:
- Calling `NewSentryProvider` twice (two `NewProviderFromConfig` calls, or a config reload)
replaces the global client. Any previously-created `SentryProvider` keeps a `hub` pointer whose
client has been swapped underneath it — events start going to the *new* DSN. If the two configs
have different environments or DSNs, events are misrouted with no error.
- Events enqueued on the old client at swap time may be dropped without flush.
- It means this "provider" abstraction is a lie: you cannot actually have two Sentry providers
with different configs in one process.
**Recommendation:** build a dedicated client with `sentry.NewClient(opts)` and bind it to an
owned `sentry.NewHub(client, scope)` rather than touching the global. That also makes `Close()`
able to genuinely release resources.
### 3. Coarse, additive shutdown flush (Medium, Slowness)
`sentry.go:125-128`
```go
func (s *SentryProvider) Flush(timeout int) bool {
return sentry.Flush(time.Duration(timeout) * time.Second)
}
```
`timeout` is an `int` interpreted as whole seconds — the interface (`interfaces.go:30`) cannot
express 500 ms. `Close()` (`sentry.go:131-134`) then runs a *second* `sentry.Flush(2s)`.
`pkg/logger.CloseErrorTracking` (`logger.go:69-75`) calls `Flush(5)` then `Close()`, so a graceful
shutdown blocks for **up to 7 seconds** in this package alone, before the HTTP drain and DB close
budgets in `pkg/server`. If the Sentry endpoint is unreachable (the common case during an
outage — which is when you are restarting) both flushes run to full timeout.
Note `Flush` also flushes the *global* client, not `s.hub`'s, which is the same object today only
because of finding 2.
**Recommendation:** change the interface to `Flush(context.Context) bool` or
`Flush(time.Duration) bool`; have `Close` not re-flush; and pass the server's shutdown deadline
through instead of hardcoding 5.
### 4. Whole stack trace stringified into `extra` on every panic (Medium, Slowness)
`sentry.go:117-119`
```go
if stackTrace != nil {
extraCtx["stack_trace"] = string(stackTrace)
}
```
The caller (`pkg/logger.CatchPanicCallback`, `HandlePanic`) already produced the trace via
`debug.Stack()`. Here it is copied again into a string and shipped as a context field. Per
recovered panic that's two full copies of a multi-kilobyte trace plus a network event. With
panics recovered rather than fatal on the request path, a reliably-panicking input is a cheap
amplification primitive (see `audit/pkg/logger.audit.md` finding 5).
Sentry also truncates large context values server-side, so much of this payload is wasted.
**Recommendation:** put the trace in `Exception[0].Stacktrace` as structured frames (which Sentry
groups and displays properly) rather than a blob in `extra`, and cap the byte length.
### 5. `AttachStacktrace: true` hardcoded (Low, Security)
`sentry.go:35`. Not configurable. Every event carries absolute source paths, package layout and
function names of the build. That's mostly a reconnaissance leak to whoever can read the Sentry
project rather than to the internet attacker, but it should be an operator choice, especially for
on-prem deployments sending to a hosted DSN.
### 6. Nil stack trace for plain errors (Low, Correctness)
`sentry.go:62`
```go
Stacktrace: sentry.ExtractStacktrace(err),
```
`ExtractStacktrace` only finds a trace if the error implements `StackTrace()`/`Callers()`
(`pkg/errors`-style). Nearly all errors in this codebase come from `fmt.Errorf`, so this returns
`nil` and the Sentry event has an exception with no frames — grouping falls back to the message
string, which (because messages embed request-specific values) fragments what should be one issue
into thousands.
**Recommendation:** fall back to `sentry.NewStacktrace()` when extraction yields nil, and set an
explicit `event.Fingerprint` derived from a stable prefix rather than the full message.
### 7. `SampleRate == 0` means "send everything" (Low, Correctness)
`factory.go:20-27` passes `cfg.SampleRate` through untouched, and `pkg/config/manager.go`
registers **no default** for `error_tracking.sample_rate`. So an operator who leaves it out gets
`0.0`, and `sentry-go@v0.46.2` `client.go:339-341` rewrites `0.0` → `1.0`.
Verified in the module cache:
```go
if options.SampleRate == 0.0 {
options.SampleRate = 1.0
}
```
Fail-open rather than fail-closed, which is arguably the right choice for an error tracker — but
it means an operator who *intends* to disable sampling by setting `0` gets the opposite, silently.
**Recommendation:** make `SampleRate` a `*float64` in the config struct, or register an explicit
default in `setDefaults`, and validate/log the effective value at init.
### 8. `factory.go` imports `pkg/config` (Low, Architecture)
`factory.go:6` — `errortracking` is imported by `pkg/logger`, which is imported by essentially
everything. Pulling `pkg/config` (and therefore `viper`) into that dependency chain means the
lowest-level logging path transitively depends on the configuration layer. It works today only
because `pkg/config` imports nothing from ResolveSpec; the first time it wants to log, there is
an import cycle.
**Recommendation:** move `NewProviderFromConfig` into `pkg/config`-adjacent wiring code (or take
a small local options struct instead of `config.ErrorTrackingConfig`) so `errortracking` stays a
leaf.
---
## What looks right
- **Concurrency is genuinely fine.** `SentryProvider` holds only an immutable `*sentry.Hub`;
`sentry.Hub` guards its own state with a mutex, and `CaptureEvent` hands off to a background
worker with a bounded queue, so it does not block the caller and does not need a lock here.
- `GetHubFromContext(ctx)` with fallback to `s.hub` (`sentry.go:53-56`, `81-84`, `103-106`) is the
correct Sentry idiom and preserves per-request scope when middleware installs a hub.
- Nil-input guards on all three capture methods (`sentry.go:47`, `76`, `98`) — a nil error, empty
message or nil recovered value is dropped rather than producing a junk event.
- `event.Contexts` is safe to index: `sentry.NewEvent()` initialises the map, so
`event.Contexts["extra"] = ...` cannot nil-panic.
- `NoOpProvider` means a disabled tracker is always safe to call — no nil checks needed at call
sites beyond the one in `pkg/logger`.
- `factory.go:15-17` correctly refuses to start with `provider: sentry` and an empty DSN rather
than silently no-oping.
## Panic handling
The package neither panics nor recovers, which is correct for its role — it is the *sink* for
panic reports, not a place that should be generating them. The nil-guards in finding "what looks
right" cover the realistic nil-deref paths. One residual: `CapturePanic` ranges over `extra`
(`sentry.go:115`) without a nil check, which is safe in Go (ranging a nil map yields zero
iterations) — noted only to confirm it was checked.
## Suggested follow-up
1. Add `BeforeSend` redaction (finding 1). This is the highest-value single change in the package.
2. Stop using the global Sentry client (finding 2) — unblocks real multi-provider support and a
meaningful `Close()`.
3. Widen `Flush` to a duration/context (finding 3) and wire it to the server shutdown budget.
4. Add tests for the Sentry path. The existing test file covers only `NoOpProvider`, severity
string mapping and interface satisfaction — `SentryProvider`'s capture methods have no
coverage at all. `sentry-go` ships a test transport that makes this straightforward.
+228
View File
@@ -0,0 +1,228 @@
# Audit: `pkg/funcspec`
| | |
|---|---|
| **Package** | `github.com/bitechdev/ResolveSpec/pkg/funcspec` |
| **Files** | `function_api.go` (1251), `parameters.go` (411), `hooks.go` (179), `hooks_example.go`, `security_adapter.go` (117) |
| **Tests** | `function_api_test.go` (1278), `hooks_test.go` (589), `parameters_test.go` (549) — 2 416 lines; `go test` and `go test -race` pass, 76.1 % statement coverage |
| **Audit date** | 2026-09-30 |
| **Axes** | thread locking/waiting, slowness, security, panic handling & logging |
| **Threat model** | hostile internet client; query string, headers and body are attacker-controlled |
| **Depth** | targeted (server-side request path; verified against source) |
## Summary
`funcspec` exposes app-defined SQL templates as endpoints. The template is
trusted; everything the client adds to it is not. The package builds SQL by
string manipulation and has two kinds of client-controlled SQL fragments
(`X-Custom-SQL-W`, `X-Custom-SQL-Or`, `sort`) that are guarded only by a keyword
denylist (`ValidSQL(..., "select")`, `function_api.go:951-980`). That is not an
injection boundary: the fragment lands inside a query that may already carry
tenant or auth predicates, and the OR path produces wrong precedence that
widens results (findings 1-3).
The auth integration is weaker than it looks. `RegisterSecurityHooks` is opt-in,
the anonymous default is `UserID 0`, and the auth hooks return an error *and*
set `Abort`, so `Execute` returns the error first and the handler answers
**400 `hook_error`**, not 401 (finding 6).
Error handling leaks: `sendError` returns the DB error text and the full SQL to
the client, and the panic recovery writes the panic value into the 500 body
(finding 5).
Resource limits are absent: default limit 100 000, no cap on `X-Limit`, no
LIMIT at all when counting is skipped, unbounded `[post_body]` read, a 15-minute
timeout, and a `COUNT(1)` over the full query on every list request (finding 8).
Positives: there are no data races under the existing tests; the `[variable]`
substitution is quote-context aware; `Content-Type` and transaction handling are
consistent; hooks run inside the transaction.
## Findings
| # | Severity | Axis | Finding |
|---|---|---|---|
| 1 | **High** | security | `X-Custom-SQL-W`, `X-Custom-SQL-Or` and `sort` are appended as raw SQL, protected only by a keyword denylist (`ValidSQL "select"`) |
| 2 | **High** | security | `sqlQryWhereOr` emits `a AND b OR (c)`; OR conditions escape the AND-ed predicates (auth/tenant filters) |
| 3 | **Medium** | security | `sqlQryWhere`/`sqlQryWhereOr` locate WHERE/GROUP BY/ORDER BY/LIMIT by substring on the lower-cased query, including inside literals, subqueries and CTEs |
| 4 | **Medium** | security | Unquoted string `X-FieldFilter` value in `ApplyFilters`; `SearchOps` keyed per column (one op per column, random order) |
| 5 | **High** | security / logging | `sendError` returns `err.Error()` and the full SQL; panic recovery writes the panic value to the 500 body |
| 6 | **High** | security / correctness | Auth hooks return an error plus `Abort`; handler replies 400 `hook_error` instead of 401; hooks are opt-in; anonymous = `UserID 0` |
| 7 | **Medium** | security | `DecodeParam` (`ZIP_`/`__`) is applied to every header/param, ignores errors and recurses without a depth limit; headers matched with `HasPrefix` |
| 8 | **High** | slowness | Default limit 100 000, no `X-Limit` cap, no LIMIT with `NoCount`/`skipcount`, unbounded `io.ReadAll` of `[post_body]`, 15-minute timeout, full `COUNT(1)` per list request |
| 9 | **Medium** | locking | `HookRegistry` map and `variablesCallback` are unsynchronized; `Register`/`Clear*` race with `Execute` |
| 10 | **Medium** | security | Dollar-quote substitution (`[post_body]`, `[user]`, `[method]`, …) skips backslash escaping; `[id_session]` substituted unquoted |
| 11 | **Low** | correctness | `Content-Range` offset comes from the `offset` query param only; header offset ignored |
| 12 | **Low** | correctness | `X-Select-Fields`/`X-Not-Select-Fields` accepted but no-ops; `sort` `-col` negates instead of DESC |
| 13 | **Low** | panic / logging | Recovery is handler-level only; `Serving: Records` logged at Info per request; hook/filter strings logged unscrubbed (X8) |
| 14 | **Low** | correctness | `BeforeResponse` runs post-commit on the pool, not the tx (see `audit/single_tran.md`) |
| 15 | **Low** | security | Security adapter hard-codes schema `public` and entity `sql_query`; per-entity rules cannot be applied |
| 16 | **Info** | testing | Regexes compiled per call (`ValidSQL`, `sqlStripStringLiterals`); no `-race` in CI (X1); no hostile-input tests for findings 1-4 |
## 1. Raw SQL fragments behind a keyword denylist — High
`ApplyFilters` (`parameters.go:283-297`) passes `X-Custom-SQL-W` and
`X-Custom-SQL-Or` through `ValidSQL(..., "select")` and splices the result into
the query. `sort` goes the same way into `ORDER BY` (`function_api.go:~226`).
The denylist (`function_api.go:964-979`) removes `;`, `--`, `/*`, `*/`, `xp_`,
`sp_` and a few keywords **followed by a space**. It is not a parser:
- Subqueries, function calls (`pg_sleep`, `pg_read_*` where permitted),
`SELECT` itself and `)` are not blocked; a `)` can close the
`COUNT(1) FROM (%s) cnts` wrapper (`function_api.go:~241`).
- Keywords are removed rather than rejected, so input can be shaped so that
removal assembles a different token.
- Whitespace variants (tab, newline) bypass the `keyword␠` patterns.
Whether the raw fragments are reachable is decided by the handler; they are
parsed whenever `ParseParameters` runs, i.e. always. Fix: drop the two headers
from the wire contract, or accept only a column/operator/value structure built
by the server; validate `sort` against `^[A-Za-z0-9_.]+( (ASC|DESC))?(,…)*$`
and ideally an allowlist of columns.
## 2. OR precedence widens results — High
`sqlQryWhereOr` (`parameters.go:381-411`) rewrites `WHERE a AND b` into
`WHERE a AND b OR (c)`. SQL evaluates `AND` first, so the result is
`(a AND b) OR c`: any row satisfying `c` is returned regardless of `a`/`b`.
Where `a` is a tenant or ownership predicate in the template, a client-supplied
OR condition (`X-SearchOr`, `X-Custom-SQL-Or`, search operator with logic OR)
returns other tenants' rows. Verified with `ParseParameters` + `ApplyFilters`
on generated headers. Fix: wrap the existing WHERE body in parentheses before
appending `OR (...)`, or build a predicate tree.
## 3. Substring-based clause location — Medium
Both helpers use `strings.Index` on `" where "`, `" group by"`, `" order by"`,
`" limit "` over the whole lower-cased query. A match inside a string literal,
a subquery, a CTE or a column alias selects the wrong insertion point, and
`wherePos > 0` decides AND-append vs. new WHERE on the first match anywhere.
`ApplyDistinct` (`parameters.go:363-378`) similarly inserts after the first
`SELECT` substring, and the ORDER BY test (`function_api.go:~224`) compares the
first `order by` to the first `from `. `sqlStripStringLiterals` exists
(`function_api.go:858`) but is not used by these helpers.
## 4. Filter handling inconsistencies — Medium
- `ApplyFilters` builds `col = value` for `X-FieldFilter` without quoting the
value (`parameters.go:248-250`), so a string value becomes a column reference
(`status = active`). `mergeHeaderParams` quotes the same filter, so the
`SqlQuery` path applies it twice with different semantics.
- `RequestParameters.SearchOps` is a map keyed by column; two operators on one
column overwrite each other and map iteration order makes the generated WHERE
non-deterministic.
## 5. Information disclosure in errors — High
- `sendError` (`function_api.go:1150-1172`) sets `Detail = err.Error()` and,
for `*common.SQLError`, `SQL` = the final statement, including the template,
substituted values and any injected fragment. Used by every failure path
(`query_failed`, `count_failed`, `hook_error`).
- Panic recovery in `SqlQueryList` (`:80-86`) and `SqlQuery` (`:433-439`) calls
`http.Error(w, fmt.Sprintf("Internal server error: %v", err), 500)`; the
panic value reaches the client. Same class as `middleware` finding 4.
Fix: log server-side, return a generic message plus a request id.
## 6. Auth hook abort returns 400, hooks opt-in — High
`RegisterSecurityHooks` (`security_adapter.go:14-55`) sets `Abort`,
`AbortCode=401` **and returns an error**. `HookRegistry.Execute`
(`hooks.go:113-137`) returns the error before it evaluates `Abort`, and the
handler maps that to `sendError(400, "hook_error", …)`
(`function_api.go:~202`). The 401 branch in the handler is only reachable for
hooks that set `Abort` without returning an error. Clients therefore see 400
with `Detail: "hook execution failed: authentication required"`.
Also: without `RegisterSecurityHooks` there is no authentication at all; a
missing user context is replaced with `UserID 0, "anonymous"`
(`function_api.go:~103`) and the request proceeds. Fix: return nil after
setting `Abort` in the auth hooks, or have the handler honour `AbortCode` when
the error wraps an abort; consider fail-closed by default.
## 7. Header/param decoding — Medium
`decodeValue` (`parameters.go:203`) calls `restheadspec.DecodeParam` and drops
the error. `DecodeParam` replaces all `ZIP_`/`__` occurrences and decodes
recursively with no depth limit, so one value can force repeated base64/gzip
work (decompression amplification, since size is not capped). Header keys are
matched with `HasPrefix`, so `X-SearchOp-<anything>` variants and unrelated
headers with the same prefix are interpreted.
## 8. Unbounded resource use — High
- `parameters.go:54` default `Limit: 100000`; `X-Limit` and `limit` accept any
positive integer.
- In `SqlQueryList` the `LIMIT`/`OFFSET` clause is added **only inside
`if !options.NoCount`** (`function_api.go:~232-251`); `NoCount` or
`X-SkipCount` returns the whole result set.
- `COUNT(1) FROM (<full query>)` runs on every list request (double execution
cost).
- `[post_body]` uses `io.ReadAll(r.Body)` (`function_api.go:913`) with no
`http.MaxBytesReader`; the body is also embedded into the SQL text.
- `context.WithTimeout(…, 15*time.Minute)` (`:91`, `:444`) holds a transaction
and pooled connection for up to 15 minutes per request.
- `ValidSQL` and `sqlStripStringLiterals` compile regexes on each call.
Fix: hard cap on limit, always apply LIMIT, cap body size, configurable timeout.
## 9. Unsynchronized registry — Medium
`HookRegistry.hooks` (`hooks.go`) is a plain map; `Register`, `Clear`,
`ClearAll` mutate it while `Execute` reads it from request goroutines. Safe
only if all registration completes before serving. `Handler.variablesCallback`
(`function_api.go:60-68`) has the same property. Fix: `sync.RWMutex` and copy-on-
read, or document and enforce "register before serve".
## 10. Dollar-quote substitution — Medium
`safeSubstituteVar` returns the raw value when the placeholder is adjacent to
`$` (`function_api.go:1044-1049`), so neither backslash nor quote escaping
applies. The tag is neutralised only for `$M$`, `$PBODY$` and the equivalents
in `replaceMetaVariables`; a caller-supplied value in a template that uses a
different tag (or `$$`) is not. `isInsideDollarQuote` inspects only the first
occurrence of the placeholder. `[id_session]` is replaced without any quoting
(`function_api.go:~900`); its source is the auth layer, but it becomes an
injection point if a session token format allows quotes.
## 11-12. Behavioural defects — Low
- `Content-Range` offset uses only `r.URL.Query().Get("offset")`
(`function_api.go:~319`) while the applied offset can come from
`X-Offset`; the reported range is wrong for header-driven paging.
- `ApplyFieldSelection` (`parameters.go:226-241`) only logs; the headers have
no effect. `sort=-col` is not converted to DESC; it is emitted as `ORDER BY
-col`, which negates the column value.
## 13. Panic handling and logging — Low
Recovery exists per handler only (no middleware-level recovery for hooks run
outside), and the stack is logged via `logger.Error`, which forwards to Sentry
unscrubbed (X8). `logger.Info("Serving: Records …")` runs on every list request.
`logger.Debug` lines include the generated filter SQL and attacker-supplied
values. Hook failures log `err` with attacker-influenced text.
## 14. `BeforeResponse` outside the transaction — Low
`BeforeResponse` executes after `RunInTransaction` returns, with
`hookCtx.Tx = h.db` (`function_api.go:~336-343`, `:~640`). A hook that writes
cannot be rolled back with the query, and a failure returns 500 after the work
committed. Tracked in `audit/single_tran.md`.
## 15. Security adapter — Low
`funcSpecSecurityContext.GetSchema()` returns `"public"` and `GetEntity()`
returns `"sql_query"` for every endpoint (`security_adapter.go:84-92`), so
column/row security rules keyed by entity cannot distinguish funcspec endpoints.
`GetModel`, `GetQuery`, `SetQuery` are stubs.
## 16. Testing — Info
Tests cover handler flow, hooks and parameter parsing. No test exercises the
hostile inputs of findings 1-4 or the 401-vs-400 outcome. There is no `-race`
job in CI (X1). The earlier note about a failing
`TestReplaceMetaVariables/Replace_[user]` no longer reproduces: the package
passes today.
## Cross-references
X1 (no `-race`), X7 (inconsistent panic handling), X8 (logger forwards to
Sentry unscrubbed), `middleware` finding 4 (panic value in body),
`audit/single_tran.md` (post-commit hooks).
+285
View File
@@ -0,0 +1,285 @@
# Audit — `pkg/logger`
- **Date:** 2026-09-29
- **Scope:** `pkg/logger/logger.go` (211 LOC, 1 file, no tests)
- **Axes:** thread locking/waiting · slowness · security · panic handling & logging
- **Threat model:** hostile internet client; request bodies, headers, params and identifiers are attacker-controlled.
## Summary
`pkg/logger` is a thin package-global wrapper over `zap.SugaredLogger` plus a fan-out to
`pkg/errortracking`. It is the single most widely imported package in the repo, so its defects
are systemic. Two classes of problem dominate: **unsynchronised global mutable state** (a real
data race between logger re-initialisation and request-path logging), and **unbounded,
unsampled, unscrubbed egress of formatted messages to a third-party error tracker** on every
`Warn`/`Error` call — which under hostile input is both a data-leak and a cost/latency
amplification channel.
There are **zero tests** in this package.
| # | Severity | Axis | Finding |
|---|----------|------|---------|
| 1 | **High** | Locking | Unsynchronised writes to `Logger` / `errorTracker` globals race with every log call |
| 2 | **High** | Security | Every `Warn`/`Error` message is shipped verbatim to Sentry — no scrubbing, no allowlist |
| 3 | **High** | Slowness | No rate limit, sampling or dedup on error-tracker fan-out; attacker-triggerable |
| 4 | **High** | Panic | `CatchPanic` swallows panics unconditionally — and both call sites are security enforcement functions (fail-open) |
| 5 | Medium | Slowness | `debug.Stack()` + full stack stringification on every recovered panic |
| 6 | Medium | Security | `log.Printf(template, args...)` fallback is a format-string sink for caller-supplied text |
| 7 | Medium | Security | No CRLF/control-char sanitisation on the stdlib fallback path → log injection |
| 8 | Medium | Correctness | `Info`/`Debug` do not strip `context.Context` args; `Warn`/`Error` do |
| 9 | Low | Correctness | `UpdateLogger` leaks the previous zap logger / file descriptor |
| 10 | Low | Correctness | No `Sync()` exported → buffered log lines lost on exit |
| 11 | Low | Slowness | `os.Getpid()` called on every log line |
| 12 | Low | Observability | `UpdateLogger` build failure degrades silently to stdlib `log` |
## Resolution status (2026-09-30)
- **#1** — Fixed (earlier race work): `stateMu` RWMutex with `getLogger`/`swapLogger`/`getErrorTracker`; the exported `Logger` var is kept for compatibility
- **#2** — Fixed: messages are scrubbed before `CaptureMessage` (URL credentials, `password=`/`token=`/`secret=`/`api_key=` values, `Bearer`/`Basic` tokens). Local logs are unchanged. Sentry `BeforeSend` and structured-field allowlisting are not done
- **#3** — Partly fixed: global token bucket (burst 50, 20/s) plus per-severity/template dedup (1s, 1024 keys). Panics are not limited. The `error_tracking.sample_rate` default (Sentry maps 0 to 1.0) is still unset in `config/manager.go`
- **#4** — Partly fixed: `CatchPanicRethrow` added. `pkg/security/provider.go:302` and `:443` still use the swallowing `CatchPanic`; left for the security audit pass
- **#5** — Fixed: stack captured with `runtime.Stack` into a 16 KiB buffer. Per-fingerprint panic rate limiting not done
- **#6** — Fixed: `Info`/`Debug` format first and fall back with `log.Printf("%s", ...)`. `gosec` was enabled separately
- **#7** — Fixed: CR/LF and other control characters are escaped on the stdlib fallback path
- **#8** — Fixed: `Info`/`Debug` strip `context.Context` args
- **#9** — Fixed: the replaced logger is synced on `UpdateLogger`
- **#10** — Fixed: `logger.Sync()` added. Not yet called from the server shutdown path
- **#11** — Fixed: PID cached in a package var
- **#12** — Partly fixed: `UpdateLoggerE` returns the build error and a failed build keeps the previous logger. `Init` still returns nothing
- Tests: `pkg/logger/logger_test.go` (run with `-race`).
---
## Findings
### 1. Unsynchronised global mutable state — data race (High, Locking)
`logger.go:14-15`
```go
var Logger *zap.SugaredLogger
var errorTracker errortracking.Provider
```
`Logger` is written by `Init` → `UpdateLogger` (`logger.go:51`) and by `UpdateLoggerPath`
(`logger.go:29`). `errorTracker` is written by `InitErrorTracking` (`logger.go:57`) and read by
`GetErrorTracker`, `CloseErrorTracking`, `Warn`, `Error`, `CatchPanicCallback`, `HandlePanic`.
Every read site (`logger.go:100`, `108`, `123`, `139`, `156`, `199`) is unguarded. There is no
mutex, no `atomic.Value`, no `sync.Once`.
- **Benign case:** everything is initialised once in `main` before goroutines start. Then it's fine.
- **Real case:** `UpdateLoggerPath` is an exported, runtime-callable API. A config reload, a
log-rotation hook, or a test helper calling it while HTTP handlers log concurrently is an
unsynchronised write to an interface value and a pointer, concurrent with reads. Under the Go
memory model this is undefined behaviour; in practice a torn interface read (type word from the
new value, data word from the old) faults.
- `CloseErrorTracking` (`logger.go:69`) does a read-check-then-use on `errorTracker` with no
guard, so a concurrent `InitErrorTracking(nil)` yields a nil-interface dereference inside
`Flush`.
**Recommendation:** store both behind `atomic.Pointer`/`atomic.Value` (or an `sync.RWMutex`),
and gate first-time init behind `sync.Once`. Run the test suite with `-race` — see finding 12 of
`audit/pkg/config.audit.md` for the same pattern in the config singleton.
### 2. Unscrubbed message egress to third-party error tracker (High, Security)
`logger.go:110-118` and `logger.go:126-134`
```go
message := fmt.Sprintf(template, remainingArgs...)
...
errorTracker.CaptureMessage(ctx, message, errortracking.SeverityError, ...)
```
*Every* `Warn` and `Error` call in the entire codebase has its fully-formatted message sent to
the configured provider (Sentry, in practice). There is no allowlist, no redaction hook, and
`pkg/errortracking/sentry.go` configures no `BeforeSend` scrubber.
Concretely, formatted error strings across `pkg/` embed: SQL fragments and bound values, DB
connection strings, schema/table/column identifiers, filter expressions built from request
input, and raw request bodies in a few handlers. Under the hostile-client threat model this is
two problems at once:
- **Outbound data leak:** secrets that appear in wrapped driver errors (DSNs, credentials from
`pq`/`pgx` connect failures) leave the trust boundary to a SaaS endpoint.
- **Attacker-controlled exfil channel:** an attacker who can shape a value that ends up in an
error message gets that value written to a third-party system — useful for exfiltrating data
read out of the DB via an induced error.
**Recommendation:** add a redaction step before `CaptureMessage`/`CapturePanic` (regex-strip
DSN/`password=`/bearer-token shapes at minimum), and set Sentry's `BeforeSend` as a second
layer. Prefer passing structured fields with an explicit allowlist over shipping the rendered
string.
### 3. No rate limiting or sampling on error-tracker fan-out (High, Slowness)
`logger.go:113`, `logger.go:129`
An unauthenticated request that reliably produces one `Error` log (a malformed filter, an unknown
column, a bad JSON body — all of which the spec handlers log at error level) becomes one Sentry
event. At even modest request rates this means:
- Sentry quota burn → a direct billing-DoS.
- `sentry-go` enqueues onto a bounded worker queue; once saturated events are dropped, so the
*real* errors are the ones lost.
- `pkg/errortracking/sentry.go:34` passes `SampleRate` straight through from config, and
`config/manager.go` sets **no default** for it. `sentry-go@v0.46.2` `client.go:339` maps
`SampleRate == 0.0` → `1.0`, so the out-of-the-box behaviour is *send 100% of events*.
**Recommendation:** default `error_tracking.sample_rate` to something < 1.0 for the message path,
and put a token-bucket or a fingerprint-dedup in front of `CaptureMessage`. Keep panics at 100%.
### 4. `CatchPanic` swallows panics unconditionally, fail-open at both call sites (High, Panic handling)
`logger.go:145-176`
```go
func CatchPanicCallback(location string, cb func(err any), args ...interface{}) func() {
...
if err := recover(); err != nil { ... if cb != nil { cb(err) } }
}
```
The recovered value is logged and then discarded. There is no variant that logs-and-re-panics
and no way for the caller to signal "this panic means state is corrupt, take the process down".
This is the right default for an HTTP handler boundary. The two current call sites are **not**
handler boundaries:
- `pkg/security/provider.go:302` — `defer logger.CatchPanic("ApplyColumnSecurity")()`
- `pkg/security/provider.go:443` — `defer logger.CatchPanic("GetRowSecurityTemplate")()`
Both are *security enforcement* functions. Swallowing a panic there means the column-security
filter or row-security template silently does not get applied, and the caller — which has no way
to learn a panic occurred, since `CatchPanic` returns nothing and sets no error — proceeds as if
security was applied. That is a fail-open security control; see
`audit/pkg/security.audit.md` for the full write-up of those two sites.
Separately: a panic while a mutex is held does not release that mutex unless an intervening
`defer Unlock` exists, so swallowing converts a crash into a permanent deadlock at any
lock-holding call site.
**Recommendation:** add `CatchPanicRethrow(location string)` for internal use and reserve the
swallowing form for the outermost request/goroutine boundary. Document which is which.
### 5. Full stack capture on every recovered panic (Medium, Slowness)
`logger.go:158` and `logger.go:197`
```go
callstack := debug.Stack()
```
`debug.Stack()` stops the world briefly and allocates; `HandlePanic` then formats the whole trace
into a string *and* ships it to Sentry. Because panics on the request path are recovered rather
than fatal (finding 4), an attacker who finds one reliably-panicking input turns each request
into a stack capture + string build + network event. That is a solid amplification factor over a
normal request.
**Recommendation:** cap the captured stack (`runtime.Stack` into a fixed 8–16 KiB buffer rather
than `debug.Stack()`'s grow-until-it-fits loop), and rate-limit identical panic fingerprints.
### 6. Format-string sink in the stdlib fallback (Medium, Security)
`logger.go:100`, `logger.go:142` (and `108`/`123` with `"%s"`, correctly)
```go
func Info(template string, args ...interface{}) {
if Logger == nil {
log.Printf(template, args...) // template is the caller's, args may be empty
```
`Info` and `Debug` pass `template` directly to `log.Printf`. If any caller ever does
`logger.Info(someUserString)` — the idiomatic-looking single-argument call — a `%s` or `%n` in
that string is interpreted as a verb, producing `%!s(MISSING)` garbage and mangled logs. Note
`Warn`/`Error` already avoid this on the fallback path by using `log.Printf("%s", message)`;
`Info`/`Debug` do not.
A grep of `pkg/` found **no** current single-argument call sites, so this is a latent API footgun
rather than a live bug — but it is one that costs one line to close.
**Recommendation:** mirror `Warn`'s shape: format first, then `log.Printf("%s", message)`.
`govet` runs by default under golangci-lint v2's standard set, and its `printf` analyser infers
wrappers like these — so once the fallback is fixed, call sites are checked at build time for
free. (Note `gosec` is *not* in `.golangci.json`'s `linters.enable` list; it appears only in the
exclusion rules. Worth enabling repo-wide.)
### 7. No log-injection sanitisation on the fallback path (Medium, Security)
On the zap path, the JSON encoder escapes newlines and control characters, so injected content
can't forge a log record. On the `Logger == nil` fallback path, `log.Printf` writes raw bytes: a
value containing `\n2026-09-29 ... level=info authorized=true` forges a plausible second log
line. Combined with finding 12 (silent degradation to the fallback path) this is reachable
without the operator noticing the encoder changed.
**Recommendation:** strip/escape `\r`, `\n` and other C0 control characters from formatted
messages before the stdlib write.
### 8. `Info`/`Debug` don't strip `context.Context` arguments (Medium, Correctness)
`extractContext` (`logger.go:79-98`) exists precisely so callers can pass a `ctx` as a trailing
variadic arg. `Warn` (`logger.go:106`) and `Error` (`logger.go:121`) call it. `Info`
(`logger.go:99`) and `Debug` (`logger.go:137`) **do not** — they pass every arg to `Sprintf`.
So `logger.Info("saved %s", name, ctx)` renders as
`saved widget%!(EXTRA *context.valueCtx=context.Background...)`, dumping the context's contents
(which in this codebase carry auth/tenant values) into the log line. That is both noise and a
minor disclosure.
**Recommendation:** call `extractContext` in all four level functions for uniform behaviour.
### 9. `UpdateLogger` leaks the previous logger (Low)
`logger.go:37-53` builds a new zap logger and overwrites `Logger` without calling `Sync()`/close
on the old one. `UpdateLoggerPath` opens a new file sink each call; repeated calls leak a file
descriptor each time and buffered lines in the old logger are lost.
### 10. No `Sync()` on shutdown (Low)
Nothing in the package exposes `Logger.Sync()`, and `CloseErrorTracking` (`logger.go:69`) flushes
only the error tracker. zap buffers writes to file sinks, so the last lines before exit — often
the interesting ones — are dropped. Add `func Sync() error` and call it from the server's
shutdown path alongside `CloseErrorTracking`.
### 11. `os.Getpid()` per log line (Low, Slowness)
`logger.go:102`, `111`, `127`, `140`, `165`, `202`. On Linux `getpid` is cached by the runtime so
this is cheap, but the PID cannot change for the life of the process — cache it in a package var
and drop six calls from the hot path.
### 12. Silent degradation when the logger fails to build (Low, Observability)
`logger.go:45-49`
```go
logger, err := config.Build()
if err != nil { log.Print(err); return }
```
`Logger` stays `nil`, so the whole process silently falls back to unstructured stdlib logging
(and thereby onto the format-string and log-injection paths of findings 6 and 7) with a single
line of warning that itself goes to stderr. A bad `logger.path` in config (unwritable directory)
triggers exactly this.
**Recommendation:** return the error from `Init`/`UpdateLogger` and let the caller decide whether
to fail startup.
---
## What looks right
- `extractContext` correctly ignores second and subsequent contexts rather than fighting over them.
- `Warn`/`Error` use `log.Printf("%s", message)` on the fallback path — the safe form.
- `HandlePanic` returns an `error` rather than swallowing, which lets callers convert a panic into
a normal error return. This is the better of the two panic idioms in the package.
- The `errortracking.Provider` indirection means a nil/noop provider is always safe to call.
## Suggested follow-up
1. Guard the two globals (finding 1) — prerequisite for running the suite under `-race`.
2. Add redaction + sampling in front of the error-tracker fan-out (findings 2, 3).
3. Split `CatchPanic` into swallow/rethrow variants and re-audit the ~60 `recover()` sites
listed in the other package audits against the split (finding 4).
4. Add a test file. Minimum: concurrent `UpdateLogger` + `Error` under `-race`, `Info` with a
`%`-bearing message, and nil-provider paths.
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
+502
View File
@@ -0,0 +1,502 @@
# Audit — `pkg/modelregistry`
- **Date:** 2026-09-29
- **Scope:** `pkg/modelregistry/model_registry.go` (381 LOC, 1 file, **no tests**)
- **Axes:** thread locking/waiting · slowness · security · panic handling & logging
- **Threat model:** hostile internet client. This package holds the `ModelRules` that
`pkg/security/hooks.go` consults to authorise read/update/create/delete, so it is **on the
authorisation path**.
## Summary
This package is the highest-risk find in the audit. It has been deliberately reworked to "never
hang" by replacing blocking `Lock`/`RLock` with **bounded `TryLock` retry loops that give up and
return a wrong answer** — and because those wrong answers are consumed by
`pkg/security/hooks.go` as authorisation decisions, the result is an **authorisation control that
fails open under lock contention**.
The comments in the file are explicit about the trade-off ("falls back to the last known value
without synchronization", "the call is a no-op") — so the hazard was known at the time of writing.
What appears not to have been traced is where those degraded results end up. They end up in
`checkModelUpdateAllowed` / `checkModelDeleteAllowed`, which treat any error as *permit*.
There are **no tests** in this package and no `-race` coverage of it anywhere.
| # | Severity | Axis | Finding |
|---|----------|------|---------|
| 1 | **Critical** | Security + Locking | `GetModel`'s "registry locked" error is consumed by `pkg/security/hooks.go` as *allow by default* → authorisation fails open under write-lock contention |
| 2 | **High** | Locking | `GetDefaultRegistry` documents and performs an unsynchronised read of `defaultRegistry` on lock-acquire failure — a data race by design |
| 3 | **High** | Locking | `SetDefaultRegistry` silently no-ops after ~20 ms of contention; caller gets no error |
| 4 | **High** | Security | `RegisterModelWithRules` is non-atomic: the model is visible with permissive `DefaultModelRules` before its real rules are applied (TOCTOU) |
| 5 | Medium | Correctness | `GetAllModels` returns an empty map, and `GetModels` silently skips whole registries, on lock-acquire failure |
| 6 | Medium | Locking | `IterateModels` invokes the caller's callback while holding `RLock` → guaranteed self-deadlock if the callback touches the registry |
| 7 | Medium | Slowness | `time.Sleep(1ms)` spin loops add up to 20 ms of latency per call and defeat mutex fairness/hand-off |
| 8 | Medium | Locking | `defaultRegistry` is read unsynchronised by six package-level functions while `SetDefaultRegistry` writes it under lock |
| 9 | Medium | Locking | Inconsistent discipline: `SetModelRules`/`GetModelRules`/`AddRegistry`/`IterateModels` use blocking locks; the rest use try-locks |
| 10 | Low | Slowness/Locking | Reflection (`TypeOf`, unwrap loop, `reflect.New`) runs while holding the registry **write** lock |
| 11 | Low | Availability | Unbounded unwrap loop: a recursive pointer type (`type T *T`) spins forever holding the write lock (**verified**) |
| 12 | Low | Panic | Package has no `recover` anywhere, and calls a caller-supplied callback under a lock (see 6) |
| 13 | Low | Security | `DefaultModelRules()` grants `CanRead/Update/Create/Delete: true` — registration without explicit rules is fully mutable |
## Resolution (2026-09-30)
Fixed in `pkg/modelregistry/model_registry.go`, `pkg/security/hooks.go`, and new
`pkg/modelregistry/model_registry_test.go` (passes under `-race`).
| # | Status | What changed |
|---|--------|--------------|
| 1 | **Fixed** | Added sentinels `ErrModelNotFound`, `ErrModelExists`, `ErrInvalidModel` (wrapped, `errors.Is`-friendly). `checkModelUpdateAllowed`/`checkModelDeleteAllowed` now allow-by-default **only** on `ErrModelNotFound`; any other error denies. Lookups can no longer return a "locked" error at all. |
| 2 | **Fixed** | `GetDefaultRegistry` uses a plain `RLock`; no unsynchronised fallback. |
| 3 | **Fixed** | `SetDefaultRegistry` uses a blocking `Lock` (cannot silently no-op); a nil registry is ignored. |
| 4 | **Fixed** | `RegisterModelWithRules` and `RegisterModel` share `registerLocked`, which writes model + rules under one lock acquisition. |
| 5 | **Fixed** | `GetAllModels`/`GetModels` use blocking locks and can no longer return empty/partial results due to contention. Signatures unchanged (`GetAllModels` is used through interfaces by resolvespec/restheadspec/openapi). |
| 6 | **Fixed** | `IterateModels` iterates a snapshot; the callback runs with no lock held (regression test re-enters the registry). |
| 7 | **Fixed** | Try-lock/sleep helpers and `lockRetry*` constants removed. |
| 8 | **Fixed** | All package-level functions go through `GetDefaultRegistry()` / `registriesSnapshot()`; `defaultRegistry` is only touched under `registriesMutex`. |
| 9 | **Fixed** | One discipline: blocking locks, snapshot-and-release, documented lock order (`registriesMutex` before a registry's mutex). |
| 10 | **Fixed** | Reflection/validation (`validateModel`) runs before the write lock is taken. |
| 11 | **Fixed** | Unwrap loop capped at 16 levels; `type T *T` now returns `ErrInvalidModel` (tested). |
| 12 | **Fixed** | Sentinel errors added. `IterateModels` recovers a callback panic per model, logs it via `logger.HandlePanic` with the model name, and continues; `validateModel` recovers reflection panics and returns `ErrInvalidModel` so registration fails closed. No lock is held during either, so the registry cannot be wedged. |
| 13 | **Accepted (decision)** | Allow-by-default retained deliberately: `DefaultModelRules()` still grants read/update/create/delete. Callers wanting restrictions must use `RegisterModelWithRules`/`SetModelRules`. |
Tests added: sentinel errors, recursive pointer type, pointer normalisation, atomic
`RegisterModelWithRules` (concurrent reader never sees permissive rules), re-entrant `IterateModels`,
cross-registry `GetModelRulesByName`, and a concurrent `-race` stress test.
Not changed: the `pkg/security` middleware-wiring question (context fast-path) remains tracked in
`audit/pkg/security.audit.md`.
---
## Findings
### 1. Authorisation fails open under lock contention (Critical, Security + Locking)
The mechanism spans two packages.
**Here**, `GetModel` conflates "not found" with "could not lock" into a single `error` return
(`model_registry.go:198-210`):
```go
func (r *DefaultModelRegistry) GetModel(name string) (interface{}, error) {
if !r.tryRLock() {
return nil, fmt.Errorf("failed to get model %s: registry locked", name)
}
defer r.mutex.RUnlock()
model, exists := r.models[name]
if !exists {
return nil, fmt.Errorf("model %s not found", name)
}
return model, nil
}
```
`GetModelRulesByName` (`model_registry.go:364-376`) uses `GetModel` as its existence probe:
```go
for _, registry := range registries {
if _, err := registry.GetModel(name); err == nil {
return registry.GetModelRules(name)
}
}
return ModelRules{}, fmt.Errorf("model %s not found in any registry", name)
```
So a `tryRLock` failure makes the registry look like it does not contain the model.
**In `pkg/security/hooks.go`**, that outcome is interpreted as *permit*
(`pkg/security/hooks.go:274-294`, and identically at `:298-318`):
```go
func checkModelUpdateAllowed(secCtx SecurityContext) error {
rules, ok := GetModelRulesFromContext(secCtx.GetContext())
if !ok {
schema := secCtx.GetSchema()
entity := secCtx.GetEntity()
var err error
if schema != "" {
rules, err = modelregistry.GetModelRulesByName(fmt.Sprintf("%s.%s", schema, entity))
}
if err != nil || schema == "" {
rules, err = modelregistry.GetModelRulesByName(entity)
}
if err != nil {
return nil // model not registered, allow by default
}
}
if !rules.CanUpdate {
return fmt.Errorf("update not allowed for %s", secCtx.GetEntity())
}
return nil
}
```
Note the context fast-path at `hooks.go:275`: if `NewModelAuthMiddleware` already put rules in the
context, the registry is not consulted and this bug does not fire. The registry fallback runs
whenever that middleware is absent or did not resolve rules — so the blast radius depends on
deployment wiring. `audit/pkg/security.audit.md` covers whether that middleware is mandatory.
`return nil` from `checkModelUpdateAllowed` means **the update is authorised**. Same for
`checkModelDeleteAllowed`. `GetModelRules(name)` for the "found" path also uses a blocking
`RLock` (`model_registry.go:253`) — so the two calls in `GetModelRulesByName` don't even use the
same locking discipline.
**Failure scenario.** A model `public.employees` is registered with `CanDelete: false`. A
concurrent `RegisterModel` (or `SetModelRules`, or `RegisterModelWithRules`) holds the write lock
for longer than `lockRetryAttempts * lockRetryDelay` = 20 ms — which is entirely achievable given
finding 10 (reflection under the write lock) and finding 7 (each waiter sleeps in 1 ms
increments, so N waiters serialise). During that window every `DELETE` request against
`public.employees` has `GetModelRulesByName` return an error, `checkModelDeleteAllowed` return
`nil`, and the delete proceeds. The model's `CanDelete: false` is not enforced.
This is remotely triggerable if any request path can cause a model registration or a rules
update; even without that, it is a straightforward race that will fire under load.
**Recommendation, in order of value:**
1. Make the security layer **fail closed**: distinguish a sentinel `ErrModelNotFound` from any
other error, and only allow-by-default on `ErrModelNotFound`. Any other error must deny.
2. Delete the try-lock scheme here entirely and use plain `RLock`/`Lock` (see finding 2 for why
the scheme does not achieve its stated goal anyway).
3. Separate the existence probe from the rules fetch so `GetModelRulesByName` takes each registry's
lock once and returns a typed "found / not found / unavailable" result.
### 2. `GetDefaultRegistry` races by design (High, Locking)
`model_registry.go:71-84`
```go
// GetDefaultRegistry returns the current default registry. It uses a
// bounded TryRLock instead of a blocking RLock so it can never hang;
// if the lock can't be acquired in time it falls back to the last known
// value without synchronization.
func GetDefaultRegistry() *DefaultModelRegistry {
for i := 0; i < lockRetryAttempts; i++ {
if registriesMutex.TryRLock() {
defer registriesMutex.RUnlock()
return defaultRegistry
}
time.Sleep(lockRetryDelay)
}
return defaultRegistry
}
```
The `return defaultRegistry` on line 83 reads a pointer that `SetDefaultRegistry`
(`model_registry.go:89-116`) writes under the write lock. The only time this path is taken is
precisely when a writer holds or is contending for the lock — i.e. the fallback executes
*exactly* in the window where the race is live. The trade is not "hang vs. slightly stale value";
it is "block for 20 ms vs. data race", and a torn/`nil` pointer read here means a nil-pointer
dereference in the caller.
The premise is also wrong: a `sync.RWMutex.RLock` that is only ever held for a map lookup cannot
"hang". The hang this was written to avoid must have had a different root cause — most likely
finding 6 (self-deadlock through `IterateModels`) or a lock-ordering inversion — and the try-lock
scheme papers over it rather than fixing it.
**Recommendation:** revert to `RLock`/`RUnlock`. If a real hang was observed, reproduce it under
`-race` and `GODEBUG=gctrace`/`SIGQUIT` stack dump; the fix belongs at the deadlock, not here.
### 3. `SetDefaultRegistry` silently no-ops (High, Locking)
`model_registry.go:90-100`
```go
acquired := false
for i := 0; i < lockRetryAttempts; i++ {
if registriesMutex.TryLock() { acquired = true; break }
time.Sleep(lockRetryDelay)
}
if !acquired {
return
}
```
The function returns no error. A caller that swaps in a registry — plausibly one with *restrictive*
`ModelRules* — has no way to learn the swap did not happen, and continues believing the new
registry is in effect. Every subsequent authorisation check consults the old registry's rules.
`GetModels` (`model_registry.go:319-329`) has the same shape and returns `nil`.
**Recommendation:** return `error` from `SetDefaultRegistry`; or (better) use a blocking `Lock`,
since this is a startup-time operation where blocking is correct.
### 4. `RegisterModelWithRules` is non-atomic (High, Security)
`model_registry.go:270-282`
```go
func (r *DefaultModelRegistry) RegisterModelWithRules(name string, model interface{}, rules ModelRules) error {
// First register the model
if err := r.RegisterModel(name, model); err != nil {
return err
}
// Then set the rules (we need to lock again for rules)
r.mutex.Lock()
defer r.mutex.Unlock()
r.rules[name] = rules
return nil
}
```
`RegisterModel` releases the write lock before returning, and it initialises the model's rules to
`DefaultModelRules()` (`model_registry.go:191-194`) — which is **permissive**:
`CanRead/CanUpdate/CanCreate/CanDelete` all `true`.
Between the two lock acquisitions, any concurrent `GetModelRulesByName` sees the model registered
with full read/update/create/delete permission, regardless of the restrictive `rules` the caller
passed. The comment "we need to lock again for rules" acknowledges the re-lock without noticing
the gap it opens.
**Failure scenario.** `RegisterModelWithRules("public.audit_log", AuditLog{}, ModelRules{CanRead:
true})` — intended read-only. A `DELETE /public.audit_log/...` that lands in the window is
authorised because `rules.CanDelete` is `true` from the default.
**Recommendation:** add an unexported `registerLocked(name, model, rules)` that writes both maps
under one lock acquisition, and build both public constructors on it. Also change the default
initialisation in `RegisterModel` to deny-by-default, or require rules at registration.
### 5. Degraded results indistinguishable from real results (Medium, Correctness)
Three functions return a plausible-looking answer when they cannot lock:
- `GetAllModels` (`model_registry.go:212-215`) — `return make(map[string]interface{})`, i.e. "the
registry is empty".
- `GetModels` (`model_registry.go:327-329`) — `return nil` on `registriesMutex` failure, and
`model_registry.go:336-338` `continue`s past any individual registry it cannot read, returning a
**partial** list with no indication of truncation.
- `GetDefaultRegistry` — finding 2.
Consumers of `GetModels`/`GetAllModels` (schema introspection, OpenAPI generation, migration
helpers) will emit a document that is missing models, and there is no error to log. Note these two
have no callers in `pkg/` today, which is the only reason this is Medium.
**Recommendation:** return `(T, error)`; never manufacture an empty-but-valid result.
### 6. `IterateModels` calls a user callback under a read lock (Medium, Locking)
`model_registry.go:307-314`
```go
func IterateModels(fn func(name string, model interface{})) {
defaultRegistry.mutex.RLock()
defer defaultRegistry.mutex.RUnlock()
for name, model := range defaultRegistry.models {
fn(name, model)
}
}
```
`fn` is arbitrary caller code running with `defaultRegistry.mutex` read-held. `sync.RWMutex` is not
reentrant, and once a writer is blocked on `Lock` it also blocks *new* readers. So:
- `fn` calling `modelregistry.RegisterModel` / `SetModelRules` → `Lock` waits for the reader, which
is the same goroutine. **Permanent self-deadlock.**
- `fn` calling `GetModel` → `tryRLock` fails for 20 ms and returns "registry locked" for every
model, which is silent nonsense rather than a deadlock (and feeds finding 1).
- `fn` doing anything slow (I/O, a DB call) holds the registry read lock for that whole duration,
blocking all registration and — via the blocked-writer rule — all other readers too.
This is the most likely original cause of the "hang" the try-lock scheme was introduced to work
around.
**Recommendation:** snapshot under the lock, release, then iterate:
```go
func IterateModels(fn func(name string, model interface{})) {
reg := GetDefaultRegistry()
snapshot := reg.GetAllModels() // takes and releases the lock
for name, model := range snapshot {
fn(name, model)
}
}
```
### 7. `time.Sleep` spin loops (Medium, Slowness)
`tryLock` (`model_registry.go:128-136`), `tryRLock` (`:140-148`), and the inline loops in
`GetDefaultRegistry`, `SetDefaultRegistry`, `GetModels`.
```go
for i := 0; i < lockRetryAttempts; i++ {
if r.mutex.TryLock() { return true }
time.Sleep(lockRetryDelay) // 1ms
}
```
Problems:
- **Latency floor.** A contended call costs a multiple of 1 ms even if the lock frees after 10 µs,
because the waiter is asleep. A blocking `Lock` would be handed the mutex in microseconds. So the
"no-hang" scheme is *slower* in the common contended case, not faster.
- **No fairness.** `sync.Mutex` has a starvation-avoidance mode that hands the lock to a waiter
queued > 1 ms. `TryLock` participates in none of it, so a try-lock waiter can be starved
indefinitely by a stream of blocking `Lock` callers (`SetModelRules`, `AddRegistry`,
`IterateModels` all still block) — see finding 9.
- **Timer churn.** 20 timer allocations per contended call.
- Sleeping in a loop scales badly: 50 concurrent callers each sleep and wake 20 times, producing
1000 needless scheduler round-trips for what a mutex does with one park/unpark.
**Recommendation:** delete the try-lock helpers. If a bounded wait is genuinely required for an
SLO, express it as `context`-aware acquisition (a buffered-channel semaphore with a `select` on
`ctx.Done()`), which gives a real deadline *and* a real error — not a silent wrong answer.
### 8. `defaultRegistry` read without the guarding mutex (Medium, Locking)
`SetDefaultRegistry` writes `defaultRegistry` (`model_registry.go:110`) under `registriesMutex`.
These read it **without** taking that mutex:
- `RegisterModel` (`model_registry.go:288`)
- `IterateModels` (`model_registry.go:308`, `311`)
- `SetModelRules` (`model_registry.go:354`)
- `GetModelRules` (`model_registry.go:359`)
- `GetDefaultRegistry`'s fallback (`model_registry.go:83`, finding 2)
A data race on the pointer, and semantically these functions may operate on the *previous* default
registry after a swap — so rules set through `SetModelRules` can land on a registry nobody consults
any more.
**Recommendation:** route every access through one accessor that takes the lock (and make
`defaultRegistry` an `atomic.Pointer[DefaultModelRegistry]` if lock-free reads are wanted — that is
the correct way to get the "never blocks" property finding 2 was reaching for).
### 9. Inconsistent locking discipline (Medium, Locking)
Within one 381-line file:
| Function | `registriesMutex` | `r.mutex` |
|---|---|---|
| `GetDefaultRegistry` | `TryRLock` + fallback | — |
| `SetDefaultRegistry` | `TryLock`, no-op on fail | — |
| `AddRegistry` (`:120`) | blocking `Lock` | — |
| `GetModelByName` (`:293`) | blocking `RLock` | via `GetModel` → `TryRLock` |
| `GetModelRulesByName` (`:365`) | blocking `RLock` | `TryRLock` then blocking `RLock` |
| `GetModels` (`:318`) | `TryRLock`, nil on fail | `tryRLock`, skip on fail |
| `RegisterModel` (`:150`) | — | `tryLock`, error on fail |
| `GetModel` (`:198`) | — | `tryRLock`, error on fail |
| `GetAllModels` (`:212`) | — | `tryRLock`, empty on fail |
| `SetModelRules` (`:237`) | — | blocking `Lock` |
| `GetModelRules` (`:252`) | — | blocking `RLock` |
| `IterateModels` (`:307`) | — | blocking `RLock` |
Four different failure behaviours for the same class of event. The mix also means the try-lock
callers can be starved by the blocking ones (finding 7), so the functions that "can never hang" are
the ones most likely to return garbage.
**Recommendation:** pick one discipline — blocking locks with snapshot-and-release — and apply it
uniformly.
### 10. Reflection under the write lock (Low, Slowness + Locking)
`RegisterModel` holds `r.mutex` (write) from `model_registry.go:151` through `:195`, and inside
that window does `reflect.TypeOf` (`:161`), the unwrap loop (`:169-171`), `reflect.New(...).Elem().Interface()`
(`:181`), and another `reflect.TypeOf` (`:185`). None of that touches `r.models`/`r.rules` and none
of it needs the lock.
This directly lengthens the window that makes finding 1 exploitable. Validate first, then take the
lock only for the two map writes.
### 11. Unbounded unwrap loop on a recursive pointer type (Low, Availability)
`model_registry.go:169-171`
```go
for modelType.Kind() == reflect.Pointer || modelType.Kind() == reflect.Slice || modelType.Kind() == reflect.Array {
modelType = modelType.Elem()
}
```
`type T *T` is legal Go, and `reflect.Type.Elem()` on it returns itself — so the loop never
terminates. **Verified experimentally:**
```go
type T *T
var x T
tt := reflect.TypeOf(x) // main.T
for tt.Kind() == reflect.Pointer { tt = tt.Elem() } // spins on main.T forever
// → "INFINITE LOOP CONFIRMED after 101 iterations, still main.T"
```
Because the loop runs with the write lock held (finding 10), this doesn't just hang one goroutine —
it wedges the registry permanently, at which point every try-lock caller starts returning
"registry locked", which via finding 1 means **authorisation fails open for the rest of the process
lifetime**.
Requires a pathological model type, so exploitability is near zero; the fix is a one-line depth cap
and it converts a permanent fail-open into an error return.
**Recommendation:** bound the loop (`for depth := 0; depth < 16 && ...; depth++`) and return an
error if the cap is hit.
### 12. No panic handling at all (Low, Panic handling)
The package contains **zero** `recover()` calls and never logs — it does not import `pkg/logger`.
For a pure data structure that is a defensible choice, with two caveats:
- `IterateModels` runs a caller callback under a read lock (finding 6). If `fn` panics, the
`defer RUnlock` does release the lock, so the registry is not wedged — that part is fine — but
the panic propagates to whatever boundary handler exists, and nothing here records which model
was being processed. A `logger`-free package can still name the model in a re-panic.
- Every failure mode in the package is reported as a `fmt.Errorf` string with no wrapping and no
sentinel values, so callers cannot distinguish them (finding 1). That is the panic/error-handling
defect that actually matters here.
**Recommendation:** define `ErrModelNotFound`, `ErrModelExists`, `ErrRegistryUnavailable` as
sentinels and wrap them, so `errors.Is` works at the security layer.
### 13. Permissive default rules (Low, Security)
`DefaultModelRules()` (`model_registry.go:24-36`) returns `CanRead`, `CanUpdate`, `CanCreate`,
`CanDelete` all `true`. `RegisterModel` applies it to any model registered without explicit rules
(`model_registry.go:191-194`), and `GetModelRules` falls back to it as well (`model_registry.go:266`).
The `CanPublic*` flags default to `false` and `SecurityDisabled` to `false`, which is right. But the
authenticated-path flags default open, so `RegisterModel(name, m)` — the form used by
`pkg/testmodels/business.go` `RegisterTestModels` and the `modelregistry.RegisterModel` convenience wrapper —
yields a fully mutable model. Combined with `pkg/security/hooks.go`'s allow-on-error, the system's
default posture at every layer is permit.
**Recommendation:** default to deny and make permissions opt-in, or at minimum log at registration
time when a model is registered without explicit rules.
---
## What looks right
- The struct-vs-pointer validation in `RegisterModel` (`model_registry.go:160-194`) is careful and
well-reasoned: it rejects `nil`, unwraps pointer/slice/array to find the base type, rejects
non-struct kinds with a message naming the original type, normalises a pointer/slice input to a
zero struct value, and re-checks the final type. The error message even tells the caller to use
`MyModel{}` instead of `&MyModel{}`. Good API ergonomics.
- Duplicate registration is rejected (`model_registry.go:156-158`) rather than silently overwriting
— important, since silent overwrite would be a rules-replacement primitive.
- `GetAllModels` returns a **copy** of the map (`model_registry.go:218-222`) rather than the
internal one, so callers cannot mutate registry state or race on it after the lock is dropped.
This is the pattern the rest of the package should follow.
- `GetModelByEntity` (`model_registry.go:225-234`) tries `schema.entity` before bare `entity`,
which is the right precedence and matches what `pkg/security/hooks.go` does.
- `GetModels` de-duplicates by name across registries (`model_registry.go:335-347`), so
registry-order precedence is consistent with `GetModelByName`'s first-match rule.
- Every `defer` for an acquired lock is correctly paired; there is no missing-`Unlock` path. The
problems here are about *which* lock discipline was chosen, not about leaking locks.
## Suggested follow-up
Ordered by risk:
1. **Make `pkg/security/hooks.go` fail closed** (finding 1). This is the single change that
converts a Critical authorisation bypass into a Medium availability issue. It does not require
touching this package.
2. **Remove the try-lock scheme** (findings 2, 3, 5, 7, 9) and fix the underlying hang by
snapshotting in `IterateModels` (finding 6).
3. **Make `RegisterModelWithRules` atomic** (finding 4).
4. Route `defaultRegistry` access through a single locked accessor or `atomic.Pointer` (finding 8).
5. Move reflection out of the write-locked region and cap the unwrap loop (findings 10, 11).
6. **Add tests.** This package has none. Priority cases: `-race` test with concurrent
`RegisterModel` + `GetModelRulesByName` asserting that rules are *never* observed as permissive
for a restrictively-registered model; a test that `GetModelRulesByName` under contention does
not return a "not found"-shaped error; `IterateModels` with a callback that calls back into the
registry (should not deadlock); sentinel-error assertions.
+121
View File
@@ -0,0 +1,121 @@
# Audit: `pkg/resolvemcp`
| | |
|---|---|
| **Package** | `github.com/bitechdev/ResolveSpec/pkg/resolvemcp` |
| **Files** | `handler.go` (901), `tools.go` (720), `cursor.go`, `oauth2.go`, `oauth2_server.go`, `annotation.go`, `hooks.go`, `security_hooks.go`, `context.go`, `resolvemcp.go` |
| **Tests** | `tools_test.go` (34), `tx_test.go` (207); `go test` passes. No hostile-input tests, no `-race` |
| **Audit date** | 2026-09-30 |
| **Axes** | thread locking/waiting, slowness, security, panic handling & logging, agent usability |
| **Threat model** | hostile or confused MCP client (LLM agent, possibly prompt-injected); tool arguments are attacker-controlled |
| **Depth** | targeted (request path, security wiring; verified against source) |
## Summary
Every model registers 4 tools + 1 resource (`read_/create_/update_/delete_<schema>_<entity>`), each with an
inlined column list, relation list and schema doc. Tool list grows 4N; context cost is
paid on every session whether or not the table is used. Replace with fixed meta tools
(see Rewrite).
Security wiring fails open in several places: model rules never reach the hooks,
`create` has no rule check, `update` skips `BeforeHandle`, update/delete skip row-level
security, and create/update write client-chosen column names. Reads have no size cap.
`resolvespec_annotate` is an unauthenticated write channel into agent-visible text.
## Findings
| # | Severity | Axis | Finding |
|---|---|---|---|
| 1 | **High** | security | Model rules set via `RegisterModelWithRules` never reach `security.Check*`: handler uses a private registry that is not `modelregistry.AddRegistry`'d; hooks look up the global list |
| 2 | **High** | security | `create` has no rule check: `CheckModelAuthAllowed` only tests `CanPublicCreate`/auth; no `BeforeCreate` hook registered, `CanCreate` never read |
| 3 | **High** | security | `executeUpdate` never fires `BeforeHandle` (create/read/delete do); auth + public-rule check skipped, only `BeforeUpdate` (`CanUpdate`) runs |
| 4 | **High** | security | Create/update data keys are not validated against model columns (`q.Value(key,…)`, `SetMap(existingMap)`): mass assignment of any column, arbitrary identifiers |
| 5 | **High** | security | Row-level security (`ApplyRowSecurity`) is wired to `BeforeRead` only; update/delete by id bypass app-level RLS (DB-level RLS via `OnTxBegin` still applies) |
| 6 | **High** | security | `resolvespec_annotate` has no auth/rule check, writes through `h.db` (outside tx, no `OnTxBegin`), any `tool_name` key; annotations are agent-facing text, so it is a prompt-injection store |
| 7 | **High** | slowness | No default/max `limit`, no max `offset`, `COUNT(*)` on every read, `[]` batch create unbounded, no statement timeout |
| 8 | **Medium** | security | `dynamicSSEHandler.pool` keyed by `Host` + `X-Forwarded-Proto` (attacker-controlled): unbounded map growth and poisoned `message` endpoint URL sent to the client |
| 9 | **Medium** | security / logging | Raw `err.Error()` (DB errors, hook errors, panic value `"internal error: %s"`) returned as tool text; `logger.Error` of the same forwards to Sentry (X8) |
| 10 | **Medium** | correctness | Update reads row, merges **json-tag keys** into `SetMap` as column names, writes every column back; breaks when json tag ≠ db column, clobbers concurrent edits (no `FOR UPDATE`) |
| 11 | **Medium** | correctness | Update ignores `nil` and `""` values: a column cannot be set to NULL or empty |
| 12 | **Medium** | correctness | Create/update commit tx 1, then run tx 2 (refetch + `AfterCreate`). Tx 2 failure returns an error for a committed write; an agent retry duplicates the insert |
| 13 | **Medium** | security | Preload relation names are passed straight to `PreloadRelation` without checking the model's relations; no depth/breadth cap |
| 14 | **Low** | security | Update/delete distinguish `record not found` from hook errors, so ids can be enumerated by error text |
| 15 | **Medium** | locking | `HookRegistry.hooks` map unsynchronized; `Register`/`Clear*` race with `Execute` (same as funcspec #9) |
| 16 | **Low** | security | Filter columns are validated by `ColumnValidator` for reads only; sort/column values are interpolated unquoted after validation (relies on validator being exact); `CustomOperators`/`ComputedColumns` unreachable from tools today, keep it that way |
| 17 | **Low** | panic | `recoverPanic` returns the panic value to the client and loses the stack; hook panics in `Execute` are not recovered before the handler-level recover |
| 18 | **Low** | agent usability | Tool names embed schema+entity (`read_public_users`); no discovery tool, so clients cannot list tables without loading every tool schema |
| 19 | **Info** | testing | No tests for auth/rule enforcement, hostile filters, key validation, limits, or `-race` |
## Details
### 1. Rules invisible to hooks (High)
`NewHandlerWithGORM/Bun/DB` call `modelregistry.NewModelRegistry()`. `security` resolves rules
via `GetModelRulesFromContext` then `modelregistry.GetModelRulesByName`, which walks the
**global** list (`registries`). The handler registry is never added, so
`ErrModelNotFound` → `CheckModelUpdate/DeleteAllowed` return `nil` (allow) and
`CheckModelAuthAllowed` falls back to "auth required, public flags ignored".
`CanUpdate=false`, `CanDelete=false` are not enforced. Fix: put rules into the
request context in `withRequestData` (`security.ModelRulesKey`) and/or `AddRegistry` on
construction.
### 2-3. Create/update gating (High)
`CheckModelAuthAllowed(op)` handles public flags only. Add `BeforeCreate` →
`CheckModelCreateAllowed` (new, mirrors update/delete), and call `BeforeHandle` at the
top of `executeUpdate`.
### 4. Column allowlist (High)
Validate every key in create/update `data` against `common.NewColumnValidator(model)`;
reject unknown keys with an error (do not silently drop on writes). Also consider a
per-model writable-column set (excluding PK, `CanPublic*`-guarded columns) for agents.
### 5. RLS on writes (High)
Run `LoadSecurityRules` + a row predicate on the update/delete pre-read query; fail the
write when the row is not visible to the user.
### 6. Annotation tool (High)
Remove from default registration or gate behind `BeforeHandle` + explicit rule. Values
returned to the agent must be treated as data, not instructions.
### 7. Limits (High)
Server config: `DefaultLimit` (e.g. 50), `MaxLimit`, `MaxOffset`, `MaxBatch`, `MaxPreloadDepth`,
per-call `context.WithTimeout`. Skip `COUNT(*)` unless requested (`with_count`).
### 8. SSE pool (Medium)
Require `Config.BaseURL` for SSE, or cap/evict `pool`, and validate `Host` against an
allowlist.
### 9/17. Error surface (Medium/Low)
Map errors to stable codes + short message; log details server-side with stack
(`logger.HandlePanic`).
### 10-12. Update/create semantics (Medium)
Build the `SET` only from validated incoming keys (column names resolved from model,
not json tags); use `NULL` for explicit null; single tx including refetch and `After*`
hooks (see `audit/single_tran.md`), or return success + warning when tx 2 fails.
## Rewrite (agreed design)
Replace per-model tools with fixed meta tools. Decisions recorded 2026-09-30:
| Decision | Choice |
|---|---|
| Functions source | Explicit registry: `Handler.RegisterFunction(name, meta, fn)`; only registered functions visible/callable |
| Old tools/resources | Removed (breaking) |
| Discovery | `list_tables`, `describe_table`, `list_functions` |
| Create | `insert_into_table` added |
| Write scope | update/delete by id **or** filters; max-rows cap, `dry_run` and confirm token apply to filter writes; id writes are single-row, no token |
| Guardrails | require id/filter, max rows affected, `dry_run`, confirm token |
| Read limits / ACL | server caps (limit, offset, preload depth); list tools filtered per caller rules |
| Identity | Authenticated caller's `UserContext`; no fixed MCP user. Endpoint guarded by OAuth / session token / API key (new `resolvespec_login_api_key`); no guest mode |
| Annotations | `resolvespec_annotate` becomes opt-in (`Config.EnableAnnotations`) and goes through `BeforeHandle` |
| Tool | Purpose |
|---|---|
| `list_tables` | registered `schema.entity` visible to caller, with allowed ops |
| `describe_table` | columns, PK, relations, writable columns, rules, limits for one table |
| `select_table` | filters/sort/columns/preloads/cursor; capped |
| `insert_into_table` | one or batch (capped); column allowlist |
| `update_table` | validated keys; guardrails |
| `delete_from_table` | guardrails |
| `list_functions` | registered functions + parameter schemas |
| `call_function` | validated args, tx + hooks |
File diff suppressed because it is too large Load Diff
+203
View File
@@ -0,0 +1,203 @@
# Audit — `pkg/testmodels`
- **Date:** 2026-09-29
- **Scope:** `pkg/testmodels/business.go` (161 LOC, 1 file, **no tests**)
- **Axes:** thread locking/waiting · slowness · security · panic handling & logging
- **Threat model:** hostile internet client. These models are registered into
`pkg/modelregistry`, which means any model here becomes a reachable entity for the spec handlers.
## Summary
Six GORM struct definitions (`Department`, `Employee`, `Project`, `ProjectTask`, `Document`,
`Comment`) used as fixtures, plus two registration helpers. No concurrency, no I/O, no panics, no
logging — so three of the four audit axes are trivially clean.
Two real issues: **all six registration errors are discarded**, and this fixture package ships in
`pkg/` (not `_test.go`, not `internal/`) where a consuming application can register test tables
into a production registry.
| # | Severity | Axis | Finding |
|---|----------|------|---------|
| 1 | Medium | Correctness | `RegisterTestModels` discards all six `RegisterModel` error returns |
| 2 | Medium | Security | Fixtures live in exported `pkg/`, registerable into a production model registry |
| 3 | Low | Security | Models are registered via `RegisterModel`, which applies permissive `DefaultModelRules` |
| 4 | Low | Correctness | `GetTestModels()` return order is unrelated to FK dependency order |
| 5 | Low | Correctness | `Document.Path` is an unconstrained filesystem path exposed as a writable API field |
---
## Findings
### 1. All registration errors discarded (Medium, Correctness)
`business.go:142-149`
```go
func RegisterTestModels(registry *modelregistry.DefaultModelRegistry) {
registry.RegisterModel("departments", Department{})
registry.RegisterModel("employees", Employee{})
registry.RegisterModel("projects", Project{})
registry.RegisterModel("project_tasks", ProjectTask{})
registry.RegisterModel("documents", Document{})
registry.RegisterModel("comments", Comment{})
}
```
`RegisterModel` returns `error` and every return value is dropped. The function itself returns
nothing, so a caller cannot detect failure either.
This matters more than usual because of how `pkg/modelregistry.RegisterModel` fails. It has two
error paths (`pkg/modelregistry/model_registry.go:151-158`):
```go
if !r.tryLock() {
return fmt.Errorf("failed to register model %s: registry locked", name)
}
...
if _, exists := r.models[name]; exists {
return fmt.Errorf("model %s already registered", name)
}
```
The first is a **transient lock-contention failure** — see `audit/pkg/modelregistry.audit.md`
finding 7, where a contended `tryLock` gives up after ~20 ms. So under concurrent registration, some
subset of these six models silently fails to register, with no error, no log, and no panic. The
process then runs with, say, `documents` and `comments` missing from the registry.
That is not merely a missing-fixture annoyance. Per `audit/pkg/modelregistry.audit.md` finding 1, an
unregistered model causes `pkg/security/hooks.go:274-294` to take the
`return nil // model not registered, allow by default` branch — so a silently-failed registration
turns into **authorisation fail-open** for that entity.
Note `errcheck` is enabled (golangci-lint v2 standard set) but `.golangci.json` excludes
`"tests?"` paths — `pkg/testmodels` does not match that pattern, so this *should* be flagged
today. Worth checking whether the linter is actually run in CI.
**Recommendation:** return `error`, and use `errors.Join` so a partial failure is reported in full:
```go
func RegisterTestModels(registry *modelregistry.DefaultModelRegistry) error {
return errors.Join(
registry.RegisterModel("departments", Department{}),
registry.RegisterModel("employees", Employee{}),
...
)
}
```
### 2. Fixtures are exported from `pkg/` (Medium, Security)
The package path is `github.com/bitechdev/ResolveSpec/pkg/testmodels`, not a `_test.go` file and not
under `internal/`. Consequences:
- The six structs and both helpers are part of ResolveSpec's **public API surface**. They are
compiled into every binary that imports anything which transitively imports this package.
- A consuming application (or a copy-pasted quickstart) that calls
`testmodels.RegisterTestModels(registry)` against its production registry makes
`departments`, `employees`, `projects`, `project_tasks`, `documents` and `comments` live entities
on the spec handlers, addressable by name. If the production database happens to have tables with
those names — `documents` and `comments` are very common names — the handlers will happily
read and write them under the permissive default rules of finding 3.
- It also means any future model added here for test convenience automatically becomes reachable.
Nothing in `pkg/` currently calls `RegisterTestModels` (only the test tree does), so this is a
packaging hazard rather than a live exposure.
**Recommendation:** move to `internal/testmodels` (blocks external import outright) or to a
`testmodels_test` package / `testdata` helper. If it must stay importable for downstream tests,
document loudly and consider a build tag.
### 3. Registered with permissive default rules (Low, Security)
`RegisterTestModels` uses `RegisterModel`, not `RegisterModelWithRules`. Per
`pkg/modelregistry/model_registry.go:191-194`, that initialises each model with
`DefaultModelRules()`, which grants `CanRead`, `CanUpdate`, `CanCreate` and `CanDelete` — see
`audit/pkg/modelregistry.audit.md` finding 13. `CanPublic*` are `false`, which is the saving grace.
If finding 2 is acted on this becomes moot; if these models are intended to stay registerable, they
should be registered read-only.
### 4. `GetTestModels()` order is not dependency order (Low, Correctness)
`business.go:152-160` returns the models in declaration order:
`Department, Employee, Project, ProjectTask, Document, Comment`.
The FK graph is not satisfied by that order. `Employee.DepartmentID → Department.ID` happens to work,
but `Document.OwnerID → Employee.ID` and `Document.ProjectID → Project.ID` mean `Document` must
follow both, and `ProjectTask.AssigneeID → Employee.ID` and `ProjectTask.ProjectID → Project.ID`
likewise. Coincidentally the declaration order does satisfy these — but nothing enforces it, and
`Employee.ManagerID → Employee.ID` is self-referential, which several migration/auto-migrate paths
handle only if the self-FK is deferred.
Also the two `many2many` joins (`department_projects`, `employee_projects`, declared at
`business.go:20`, `:45`, `:67-68`) are not in the returned list at all, so a caller using
`GetTestModels()` to drive `AutoMigrate` gets the join tables only because GORM infers them from the
tags — a Bun-based migration path (`pkg/common/adapters/database/bun.go`) would not.
**Recommendation:** document that the order is migration-safe and add a comment stating the
constraint, or return an explicitly ordered list with a test that asserts it.
### 5. `Document.Path` is an unconstrained path field (Low, Security)
`business.go:107`
```go
Path string `json:"path"`
```
No validation, no length limit, no `gorm` constraint. As a plain string column it is inert — the
risk only materialises if some handler or downstream consumer uses it to open a file, at which point
an attacker who can `POST`/`PATCH` a `Document` controls a filesystem path (`../../etc/passwd`,
`/proc/self/environ`). The same applies to `ContentType` (`business.go:105`) if it is ever echoed
into a response header unvalidated, and `Size` (`business.go:106`) which is a client-settable
`int64` that can disagree with reality.
Nothing in `pkg/` reads these fields, so this is a note about the fixture's shape rather than a
present vulnerability — but it is a bad example to ship, since fixtures get copied.
**Recommendation:** if these stay, mark `Path` as server-set (a `gorm:"->"` read-only tag, or
exclude it from the writable column set) so the fixture demonstrates the safe pattern.
---
## Axis-by-axis
- **Thread locking / waiting:** nothing to report. The package declares no goroutines, channels,
mutexes or atomics. Its only concurrency exposure is *through* `pkg/modelregistry`, covered in
finding 1 and in that package's audit.
- **Slowness:** nothing to report. `RegisterTestModels` and `GetTestModels` are O(1) with six
elements and are startup-only. The `TableName()` methods (`business.go:23`, `:49`, `:73`, `:96`,
`:119`, `:137`) return constants — no allocation, no reflection.
- **Security:** findings 2, 3, 5 — all about packaging and field shape, none about code behaviour.
- **Panic handling and logging:** the package contains no `panic`, no `recover`, and does not import
`pkg/logger`. For plain struct definitions that is correct. The one place where logging *would*
belong is the discarded errors of finding 1 — silently dropping six error returns is the
panic/error-handling defect in this package, even though no panic is involved.
## What looks right
- Struct tags are consistent and complete: `json` on every field, `gorm:"primaryKey"` on every ID,
`gorm:"uniqueIndex"` on the natural keys (`Department.Code`, `Employee.Email`, `Project.Code`),
and explicit `foreignKey`/`references` on every relation rather than relying on GORM's inference.
That makes these fixtures genuinely useful for exercising the relation-expansion paths in
`pkg/restheadspec` and `pkg/resolvespec`.
- `omitempty` on every relation field prevents empty relation arrays from bloating responses — which
matters, because these fixtures are what the handler tests measure payloads against.
- Nullable FKs are correctly modelled as `*string` (`Employee.ManagerID` `business.go:35`,
`Document.ProjectID` `business.go:109`) rather than empty-string sentinels.
- The self-referential manager/reports pair (`business.go:43-44`) and the two `many2many` relations
give reasonable coverage of the harder relation shapes — a genuinely well-chosen fixture set for
the recursive-preload logic audited in `audit/pkg/restheadspec.audit.md`.
- `TableName()` is defined on the value receiver for all six, so it works whether a value or a
pointer is passed — which matters given `pkg/modelregistry.RegisterModel` normalises pointers to
values.
## Suggested follow-up
1. Return and check errors from `RegisterTestModels` (finding 1). One-line-per-call change, and it
closes a silent path to authorisation fail-open.
2. Decide whether this package belongs in `pkg/` at all (finding 2). `internal/testmodels` is the
low-effort fix.
3. Confirm `golangci-lint` runs in CI and that `errcheck` flags `business.go:143-148` — if it does
not, the exclusion patterns in `.golangci.json` need review, since this is exactly the class of
bug it exists to catch.
+286
View File
@@ -0,0 +1,286 @@
# Audit — `pkg/tracing`
- **Date:** 2026-09-29
- **Scope:** `pkg/tracing/tracing.go` (146 LOC, 1 file, **no tests**)
- **Axes:** thread locking/waiting · slowness · security · panic handling & logging
- **Threat model:** hostile internet client. Span names and attributes here are built directly from
request-controlled data (method, path, full URL, Host header).
## Summary
A thin OpenTelemetry wrapper: `InitTracer`, an HTTP middleware, and helpers. The abstraction is
fine; the **hardcoded choices** are the problem. Three of them are not configurable at all and each
is wrong for a production, internet-facing deployment:
- `otlptracegrpc.WithInsecure()` — trace export is **plaintext**, with a source comment admitting it.
- `sdktrace.AlwaysSample()` — **100% of requests** are traced, with no sampling knob in config.
- `semconv.HTTPURLKey.String(r.URL.String())` — the **full URL including query string** is exported.
Combined: every request's full URL is shipped unencrypted to a collector, and an attacker sets the
export volume. Span names are also built from raw paths, giving unbounded cardinality.
`config.TracingConfig` (`pkg/config/config.go:86-91`) exposes only `Enabled`, `ServiceName`,
`ServiceVersion` and `Endpoint` — there is no field for TLS or sample rate, so these cannot be fixed
by configuration alone.
| # | Severity | Axis | Finding |
|---|----------|------|---------|
| 1 | **High** | Security | `WithInsecure()` hardcoded — traces exported in plaintext, not configurable |
| 2 | **High** | Security | Full URL **including query string** exported as a span attribute |
| 3 | **High** | Slowness | `AlwaysSample()` hardcoded — 100% trace volume, attacker-controlled, no sampling config |
| 4 | Medium | Slowness | Span name is `method + " " + r.URL.Path` — unbounded cardinality from raw path IDs |
| 5 | Medium | Locking | `tracer` global written by `InitTracer`, read unsynchronised by `Middleware`/`StartSpan` |
| 6 | Medium | Observability | `Middleware` records no HTTP status and no error status — spans never show failures |
| 7 | Medium | Panic | `Middleware` does not recover; a downstream panic leaves the span unmarked (`Unset` status) |
| 8 | Low | Slowness | `InitTracer` has no timeout/deadline on exporter or resource creation |
| 9 | Low | Maintenance | `semconv/v1.4.0` (2021) — deprecated attribute names modern collectors no longer index |
| 10 | Low | Security | `SetAttributes`/`AddEvent` pass caller data through with no size or cardinality limit |
## Resolution (2026-09-30)
Fixed in `pkg/tracing/tracing.go`, `pkg/config` (`TracingConfig`, defaults), the package README, and new
`pkg/tracing/tracing_test.go` (passes).
| # | Status | What changed |
|---|--------|--------------|
| 1 | **Fixed** | TLS is the default. `Config` gains `Insecure`, `TLSConfig` and `Headers` (OTLP auth). `tracing.insecure` added to `pkg/config`. **Breaking:** plaintext collectors now need `Insecure: true`. |
| 2 | **Fixed** | Query string and `Host` are no longer exported; attributes are method, `url.path`, scheme, `http.route`, status. `TLSConfig` has no config-file key (code only). |
| 3 | **Fixed** | `ParentBased(TraceIDRatioBased(rate))`; `SampleRate` defaults to 0.1, validated to [0,1]; `tracing.sample_rate` added to `pkg/config`. |
| 4 | **Fixed** | Span name is `METHOD <route template>` from `Request.Pattern`, `<unmatched>` otherwise. `MiddlewareWithRoute(fn)` supports other routers. |
| 5 | **Fixed** | `tracer` is an `atomic.Pointer`; a second `InitTracer` returns an error; the shutdown func resets state. |
| 6 | **Fixed** | Response writer wrapped; `http.response.status_code` recorded, 5xx sets Error status. Preserves `Flush`/`Unwrap`. |
| 7 | **Fixed** | Panics are recorded (`RecordError`, Error status) and re-raised so the panic middleware still responds. Must be installed inside the panic middleware; actual order in `pkg/server` not verified. |
| 8 | **Fixed** | `InitTracerContext(ctx, cfg)` with `InitTimeout` (default 10s); `InitTracer` retained as a wrapper. Exporter is shut down if resource creation fails. |
| 9 | **Fixed** | Moved to `semconv/v1.26.0`. |
| 10 | **Fixed** | `AttributeValueLengthLimit` set via `WithRawSpanLimits` (`AttributeValueLimit`, default 1024). |
Tests added: query redaction and route naming, unmatched route, 5xx status, panic recorded and re-raised,
double-init and invalid sample rate.
---
## Findings
### 1. `WithInsecure()` hardcoded (High, Security)
`tracing.go:38-42`
```go
client := otlptracegrpc.NewClient(
otlptracegrpc.WithEndpoint(config.Endpoint),
otlptracegrpc.WithInsecure(), // Use WithTLSCredentials in production
)
```
The comment names the fix and the code does not implement it, and — critically — `Config`
(`tracing.go:21-27`) has no field to express it:
```go
type Config struct {
ServiceName string
ServiceVersion string
Endpoint string
Enabled bool
}
```
So there is **no supported way** to enable TLS on trace export short of editing this file. Every
span — carrying the full request URL per finding 2 — crosses the network in cleartext, and the
collector endpoint is unauthenticated (no OTLP headers/bearer token option either), so anything that
can reach it can also *inject* fabricated spans.
**Recommendation:** add `Insecure bool`, `TLSConfig *tls.Config` and `Headers map[string]string` to
`Config` (and the matching `tracing.*` keys to `pkg/config`), default to TLS on, and require an
explicit opt-in for insecure. Wire `otlptracegrpc.WithTLSCredentials` / `WithHeaders`.
### 2. Full URL with query string exported (High, Security)
`tracing.go:95-103`
```go
ctx, span := tracer.Start(ctx, r.Method+" "+r.URL.Path,
trace.WithSpanKind(trace.SpanKindServer),
trace.WithAttributes(
semconv.HTTPMethodKey.String(r.Method),
semconv.HTTPURLKey.String(r.URL.String()), // <- full URL, query string included
semconv.HTTPTargetKey.String(r.URL.Path),
semconv.HTTPSchemeKey.String(r.URL.Scheme),
semconv.NetHostNameKey.String(r.Host),
),
)
```
`r.URL.String()` includes `RawQuery`. For this API the query string is where the interesting data
lives: filter expressions, column lists, and — for any client that passes credentials as a query
parameter (`?api_key=`, `?token=`, signed-URL style parameters) — secrets. All of it lands in the
tracing backend, and per finding 1 it gets there in plaintext.
Note `HTTPTargetKey` is also set to `r.URL.Path`, so the *useful* part is already captured
separately; `HTTPURLKey` adds only the sensitive part.
Secondary: `r.Host` comes from the `Host` header, which is client-controlled and unvalidated here —
so an attacker can pollute the `net.host.name` dimension with arbitrary values (cardinality blowup,
and log/dashboard spoofing).
**Recommendation:** export a redacted URL (scheme + host + path, query keys only or dropped
entirely). OTel's own guidance is to strip or redact query parameters for exactly this reason.
### 3. `AlwaysSample()` hardcoded (High, Slowness)
`tracing.go:61-65`
```go
tp := sdktrace.NewTracerProvider(
sdktrace.WithBatcher(exporter),
sdktrace.WithResource(res),
sdktrace.WithSampler(sdktrace.AlwaysSample()),
)
```
Every request produces a recorded, exported span. There is no `SampleRate` in `Config` and no
`tracing.sample_rate` key in `pkg/config/manager.go`'s defaults, so this is not tunable.
Under the hostile-client threat model the request rate — and therefore the span rate, the batch
queue pressure, the serialisation cost and the outbound bandwidth — is set by the attacker. Each
request pays span allocation, attribute encoding (including the full URL string), and a share of
batch export. When the batch queue fills, the SDK drops spans, so a flood also destroys the
observability you need to see the flood.
**Recommendation:** default to `sdktrace.ParentBased(sdktrace.TraceIDRatioBased(rate))` with a
configurable rate (e.g. 0.01–0.1), keeping `AlwaysSample` available for development. `ParentBased`
also means an upstream sampling decision is respected, which `AlwaysSample` currently overrides.
### 4. Unbounded span-name cardinality (Medium, Slowness)
`tracing.go:95` — the span name is `r.Method + " " + r.URL.Path`.
This API's paths embed identifiers (`/api/public/employees/7f3c…`, `/api/<schema>/<entity>/<id>`), so
each distinct ID becomes a distinct span name. Consequences:
- Tracing backends index on span name; unbounded distinct names is the classic cardinality-explosion
cost bomb (and in some backends, a hard limit that starts rejecting data).
- It violates the OTel HTTP convention, which requires a **low-cardinality route template**
(`GET /api/{schema}/{entity}/{id}`), with the concrete value in `http.route`/attributes.
- It is attacker-driven: requests to random paths — including 404s — each mint a new span name.
**Recommendation:** derive the name from the matched route pattern. `pkg/server`'s router
(chi/mux/gin, see `audit/pkg/server.audit.md`) exposes the route template after matching; use it, and
place this middleware after the router so the pattern is available. Fall back to
`r.Method + " " + "<unmatched>"` rather than the raw path.
### 5. Unsynchronised `tracer` global (Medium, Locking)
`tracing.go:19`
```go
var tracer trace.Tracer
```
Written at `tracing.go:77` (`tracer = tp.Tracer(config.ServiceName)`), read at `tracing.go:86`,
`:95` (`Middleware`) and `:116`, `:119` (`StartSpan`). No mutex, no `atomic.Value`.
Same pattern as `pkg/logger`'s `Logger` global (see `audit/pkg/logger.audit.md` finding 1). Benign if
`InitTracer` runs once before any request is served; a race the moment tracing is re-initialised at
runtime. `InitTracer` is exported and callable at any time, and calling it twice also leaks the
first `TracerProvider` (nothing shuts it down) — its batch processor goroutine and gRPC connection
stay alive for the life of the process.
Note the nil checks at `:86` and `:116` are the read-half of the race: a goroutine can observe a
non-nil-but-torn interface value.
**Recommendation:** `atomic.Pointer` or a `sync.Once`-guarded init; return an error from a second
`InitTracer` call, or shut down the previous provider first.
### 6. No HTTP status or error status on spans (Medium, Observability)
`Middleware` (`tracing.go:84-112`) never wraps `w`, so it cannot observe the status code. It sets no
`semconv.HTTPStatusCodeKey` and never calls `span.SetStatus`. Every span therefore has status
`Unset`, which tracing backends render as "OK".
The practical effect: you cannot find failing requests in the traces. A 500-storm and a healthy
period look identical in the span data, which defeats the main reason to run tracing on an
internet-facing service.
**Recommendation:** wrap the `ResponseWriter` to capture the status, set
`semconv.HTTPStatusCodeKey.Int(status)`, and `span.SetStatus(codes.Error, ...)` for 5xx.
### 7. No panic handling in the middleware (Medium, Panic handling)
`Middleware` has `defer span.End()` (`tracing.go:106`) but no `recover()`. If `next.ServeHTTP`
panics:
- The `defer span.End()` **does** run, so no span is leaked — that part is correct.
- But the span is ended with status `Unset` and no exception event, so the panic is invisible in the
trace. The one place a trace would be most valuable records nothing.
- The panic propagates up to whichever handler is outermost. Whether that is
`pkg/middleware/panic.go` depends on middleware ordering — if `tracing.Middleware` is installed
*outside* the panic middleware, the panic escapes to `net/http`'s per-connection recovery, which
kills the connection and logs to the default logger, bypassing `pkg/logger` and the error tracker
entirely. See `audit/pkg/middleware.audit.md` and `audit/pkg/server.audit.md` for the actual order.
This package does not import `pkg/logger` at all, so nothing here can be logged.
**Recommendation:** recover, record `span.RecordError` + `span.SetStatus(codes.Error, …)`, then
re-panic so the dedicated panic middleware still handles the response. Document the required
middleware order.
### 8. No deadline on initialisation (Low, Slowness)
`tracing.go:36` uses `ctx := context.Background()` for both `otlptrace.New` (`:44`) and
`resource.New` (`:51`). `otlptracegrpc` does not block on connect by default, so this is unlikely to
hang today — but `resource.New` with detectors can perform network calls (cloud metadata endpoints),
and an unreachable metadata service is a classic multi-second startup stall. `InitTracer` should
accept a `context.Context` from the caller so startup has a deadline.
### 9. `semconv/v1.4.0` (Low, Maintenance)
`tracing.go:15` pins the 2021 semantic conventions. `http.method`, `http.url`, `http.target`,
`http.scheme`, `net.host.name` were all renamed in v1.20+ (`http.request.method`, `url.full`,
`url.path`, `url.scheme`, `server.address`). Current collectors, dashboards and backend
auto-instrumentation views key off the new names, so these spans will not populate standard HTTP
dashboards.
### 10. No limits on caller-supplied span data (Low, Security)
`StartSpan`, `AddEvent`, `SetAttributes` (`tracing.go:115-145`) forward caller attributes verbatim.
If any caller passes request-derived values (a filter expression, a row payload), span size is
attacker-influenced. The SDK's default limits (128 attributes, 128 events) cap the count but not the
*value* length — a 1 MB string attribute is accepted.
**Recommendation:** set explicit `sdktrace.WithSpanLimits` including `AttributeValueLengthLimit`.
---
## What looks right
- **Disabled path is genuinely free.** `InitTracer` with `Enabled: false` (`tracing.go:31-34`)
returns a no-op shutdown func and never builds an exporter, so a disabled deployment pays nothing
and cannot leak.
- **Nil-tracer guards everywhere.** `Middleware` (`tracing.go:86-89`) passes through untouched and
`StartSpan` (`tracing.go:116-118`) returns the incoming context plus the context's (no-op) span.
So a partially-initialised process degrades safely rather than nil-panicking — a pattern
`pkg/logger` gets right too.
- **Context propagation is correct.** `Extract` from `propagation.HeaderCarrier(r.Header)`
(`tracing.go:92`), a composite `TraceContext` + `Baggage` propagator (`tracing.go:72-75`), and
`r = r.WithContext(ctx)` (`tracing.go:109`) before calling `next` — the span context actually
reaches downstream handlers, which is the part most hand-rolled middlewares get wrong.
- `SpanKindServer` is set correctly (`tracing.go:96`).
- `WithBatcher` rather than a simple/sync span processor (`tracing.go:62`) — export does not block
the request path.
- `InitTracer` returns `tp.Shutdown` (`tracing.go:80`), giving the caller a real flush-on-shutdown
hook with a caller-supplied context, which is better than the fixed-timeout pattern in
`pkg/errortracking` (see that audit, finding 3).
- `RecordError` nil-guards (`tracing.go:140-143`) so `RecordError(ctx, nil)` is a no-op.
## Suggested follow-up
1. Extend `Config` with `Insecure`, TLS credentials, OTLP headers and `SampleRate`; add the matching
keys to `pkg/config` (findings 1, 3). These cannot be fixed without an API change, so they should
go together.
2. Redact the query string from exported attributes (finding 2).
3. Move to route-template span names, which requires positioning the middleware after routing
(finding 4).
4. Capture status code and panics in the middleware (findings 6, 7).
5. Guard the `tracer` global (finding 5) and upgrade `semconv` (finding 9).
6. Add tests: this package has none. A tracetest/in-memory exporter makes assertions on span name,
attributes and status straightforward, and would have caught findings 2, 4 and 6.
+122
View File
@@ -0,0 +1,122 @@
# Single transaction per request — plan
## Goal
- Every DB statement and every hook that touches the DB in one request runs on **one transaction / one connection**.
- Hooks never receive the raw pool (`h.db`).
- Fixes: RLS GUCs (`set_config(..., true)`) lost on reads/creates/deletes; extra pool connections; select-then-write races.
## Why
- `set_config(..., true)` is transaction-local. A hook on the pool, or a query on another pool connection, never sees it → RLS returns 0 rows / 42501.
- Each un-transacted call takes its own pool connection → bursts with a small pool (see `dbtrace`).
- Already fixed: read/create hooks in `resolvespec` + `restheadspec` (commit `47708fc`, tag >= v1.1.28). Consumers on older tags still show the bug.
## Current state (verified by reading code; not yet by `dbtrace`)
| Spec | Read | Create | Update | Delete |
|---|---|---|---|---|
| restheadspec | tx; `AfterRead` post-commit on pool | tx; `AfterCreate` post-commit on pool | tx; re-fetch + `BeforeScan` post-commit on pool (`:1667-1674`) | **single: no tx, hook + select + delete on pool (`:1945-1994`)**; batch: tx, per-item `BeforeDelete` inside |
| resolvespec | tx | tx | tx; re-fetch on pool (`:1297, 1449, 1602`) | **single: hook + select + delete on pool (`:1654, 1794, 1806`)**; batch: one `BeforeDelete` before tx, none per item |
| websocketspec | **pool** (`:563-672`) | **pool** (`:708`) | **pool** (`:744`) | **pool** (`:757`) |
| mqttspec | **pool** (`:674-789`) | **pool** (`:838`) | **pool** (`:875`) | **pool** (`:889`) |
| resolvemcp | **pool** (`:253`) | single: **pool** (`:445`); batch: tx | tx | tx |
| funcspec | tx; `BeforeResponse` post-commit on pool (`:337, 640`) | — | — | — |
- Correction to earlier note: "no transactions" in mqttspec/websocketspec/resolvemcp-read is a gap for this problem, not a non-issue.
- `BeforeHandle` runs before any tx by design (auth + model checks, `PreloadSecurityRules`). Keep it DB-free except security preload (own connection, cached).
## In-tx hook coverage today (verified)
- Already in tx with `Tx: tx`: `BeforeRead`, `BeforeCreate`, `BeforeUpdate`, `BeforeScan` (read/create/update, both specs); restheadspec batch delete `BeforeDelete` + `AfterDelete`.
- **Not in tx:** single `BeforeDelete`/`AfterDelete` (both specs), resolvespec batch delete (no per-item hook), all `After*` post-commit, all websocketspec/mqttspec hooks, resolvemcp read/single create.
- Gap beyond coverage: no single guaranteed "tx opened" point. User/RLS stamping would have to be repeated in each `Before*` hook and is missed by any path without one (e.g. resolvespec batch delete). `OnTxBegin` closes this: fires once per tx, first, for read/insert/update/delete and for the second short tx.
## Scope
- `OnTxBegin` + `runInTx` apply to **all six**: resolvespec, restheadspec, websocketspec, mqttspec, resolvemcp, funcspec.
- Each spec has its own `HookType` (resolvespec, restheadspec, websocketspec, resolvemcp, funcspec); mqttspec aliases websocketspec, so it inherits the constant but needs its own handler wiring.
- Same semantics everywhere: fires once per tx, first, for read/insert/update/delete and the second short tx; failure aborts + rolls back, nothing leaked.
- Each spec's `security_hooks.go` registers the user/RLS stamping on `OnTxBegin`.
- Shared helper preferred over six copies: one small function in `pkg/common` (begin tx, set `Tx`, call spec-supplied begin callback), each spec passes its own hook executor.
## funcspec (different)
- Custom SQL handlers (`SqlQuery`, `SqlQueryList`), no CRUD, no model registry; one tx per request already (`:195`, `:561`).
- Already in tx: `BeforeQuery`/`BeforeQueryList`, `BeforeSQLExec`, `AfterSQLExec`, `AfterQuery`/`AfterQueryList`, plus `BeforeOp`.
- `BeforeOp` = generic pre-hook via `ExecuteBeforeOp`, fires before every `Before*` in the tx; but it fires **twice** per tx (query hook + `BeforeSQLExec`), so it is not a once-per-tx point.
- Only gap: `BeforeResponse` runs post-commit with `Tx = h.db` (`:337`, `:640`).
- Applies from this plan: once-per-tx `OnTxBegin` (stamping user/RLS before any SQL, incl. hook-mutated SQL), `BeforeResponse` on a tx, fail-closed abort.
- Does not apply: delete/insert/update phases, second re-fetch tx (no re-fetch; SQL is user-defined), `BeforeHandle` preload.
- Decided: add a real `OnTxBegin`. `BeforeOp` unchanged.
## Common interface (`pkg/common`, new `txhook.go`)
- Precedent: `security.SecurityContext` + per-spec `newSecurityContext(hookCtx)` adapter. Same pattern here.
- `common.TxHookName` = `"on_tx_begin"`: one shared string; each spec declares `OnTxBegin HookType = common.TxHookName` (HookTypes are per-spec types, so the constant value is shared, not the type).
- `common.TxContext` interface, implemented by each spec's `HookContext` via a small adapter: `GetContext()`, `GetTx()`, `SetTx(common.Database)`, plus `Abort` accessors for the abort path.
- `common.RunRequestTx(ctx, db, tc TxContext, onBegin func() error, body func(tx common.Database) error) error`: `RunInTransaction` -> `tc.SetTx(tx)` -> `onBegin()` (spec passes `registry.Execute(OnTxBegin, hookCtx)`) -> `body(tx)`. `onBegin` error or abort = return error = rollback.
- Shared stamping: one function in `pkg/security` taking `SecurityContext` + `common.Database` (sets tx-local user/RLS); each spec's `RegisterSecurityHooks` registers it on `OnTxBegin`. No per-spec copies of the logic.
- Each spec keeps its own registry/`HookContext`; only the tx lifecycle and stamping are shared.
- Out of scope: unifying the six `HookContext` / registry types.
## Design
1. **`OnTxBegin` hook** (new `HookType`, all specs). Runs first inside every tx the handler opens; gets `tx` in `hookCtx.Tx`. RLS stamping is registered once there; reads owner/tenant from request context.
2. **`runInTx` helper per handler**: wraps `RunInTransaction`, sets `hookCtx.Tx = tx`, fires `OnTxBegin`, runs the body. All handler paths use it; no path passes `h.db` to a hook.
3. **Post-commit work** (`After*`, update re-fetch, `BeforeResponse`): run in a second short `runInTx` (so `OnTxBegin` re-applies). Not inside the main tx.
4. **Delete**: hook → select → delete in one tx; 404 on no row; cache invalidation after commit.
5. **Backward compat**: hooks keep the same names/order; only `hookCtx.Tx` changes from pool to tx. `OnTxBegin` is additive.
## Decisions (settled)
- Insert/update: re-fetch + `AfterCreate`/`AfterUpdate`-style post-commit work run in a **second short tx** (must see trigger changes). Only insert/update; read and delete have no second tx.
- Update re-fetch is a plain SELECT in that second tx. No `RETURNING`.
- `OnTxBegin` failure aborts the whole request, rolls back, returns an error with no detail leaked to the client.
## Open
- Consumer's ResolveSpec version: confirm it is >= v1.1.28 (read/create already in tx). Not blocking.
## Phases
| # | Status | Change | Files | Notes |
|---|---|---|---|---|
| 0 | DONE | Baseline: enable `dbtrace` on testserver, record `tx/tx_queries/pooled/raw` per op | `cmd/testserver`, `pkg/dbtrace` | pooled > 0 on write ops = the gaps above |
| 1 | DONE | Delete in one tx (single + batch, per-item hooks inside tx) | `resolvespec/handler.go`, `restheadspec/handler.go` | fixes 2 pool connections + race + RLS |
| 2 | DONE | `OnTxBegin` hook type + `runInTx` helper | `common/txhook.go`, `*/hooks.go`, `*/handler.go` | resolvespec + restheadspec; other specs in P4-6 |
| 3 | DONE | Insert/update post-commit hooks + re-fetch in second short `runInTx` (select only) | restheadspec `:1005, 1467, 1667-1674`; resolvespec `:1297, 1449, 1602` | per decision above |
| 4 | DONE | websocketspec + mqttspec: wrap read/create/update/delete in `runInTx` | `websocketspec/handler.go`, `mqttspec/handler.go` | mqttspec aliases websocketspec hooks; confirm `OnTxBegin` alias |
| 5 | DONE | resolvemcp: read + single create in tx | `resolvemcp/handler.go:253, 445` | verify batch/update/delete hooks run inside tx |
| 6 | DONE | funcspec: `OnTxBegin` (or once-per-tx `BeforeOp`), `BeforeResponse` via `runInTx` | `funcspec/function_api.go:337, 640` | |
| 7 | DONE | Security hooks: register RLS stamping on `OnTxBegin`; document | `pkg/security/*`, README | |
## Progress
- DONE P0: baseline via `dbtrace` on real Postgres (commit `cd96404`): create/read/delete `pooled=0`; update `pooled=1` (re-fetch) = P3 target. websocketspec/mqttspec/resolvemcp not measured.
- DONE P1: single + batch delete in one tx (resolvespec, restheadspec). Not done: per-item `BeforeDelete` in resolvespec batch (behavior change, deferred).
- DONE infra: `sqlmock` delete tx tests (both specs); compose test server + `scripts/testserver-smoke.sh` (podman first); testmodels ids now serial.
- NOTE: restheadspec single delete still does the lookup before `BeforeDelete`; safe once `OnTxBegin` (P2) exists. An `AfterDelete` failure now rolls the delete back.
- DONE P2 (resolvespec + restheadspec): `common.TxHookName`, `common.TxContext` (`SetTx` only; no abort/context accessors needed since `Execute` already returns an error on abort), `common.RunRequestTx`; per-spec `OnTxBegin`, `HookContext.SetTx`, `Handler.runInTx`. Every `RunInTransaction` in both handlers now goes through it. Tests: `pkg/*/on_tx_begin_test.go` (once, first, on tx, failure rolls back). Not yet: the post-commit second tx (P3) and the security stamping registration (P7).
- DONE P3: restheadspec update re-fetch + `BeforeScan` + `AfterUpdate` and `AfterCreate` run in a second short `runInTx`; resolvespec update re-fetches (single, both batch paths) run in a second short `runInTx`. Fixed the pool reads inside the first tx (resolvespec single/batch update existing-record select, restheadspec update existence select) to use `tx`. Tests: `pkg/*/update_tx_test.go` (restheadspec uses the bun adapter; the pgsql adapter cannot build model-based updates).
- NOTE: resolvespec fires no `AfterCreate`/`AfterRead`/`AfterUpdate`-post-commit hooks other than `AfterUpdate` inside the tx; nothing more to move there.
- OPEN: restheadspec `AfterRead` still runs post-commit with `Tx = h.db` (`:1004`); decision says read has no second tx. Needs a call: run it inside the read tx, or in a short second tx.
- DONE P4: websocketspec + mqttspec. `OnTxBegin` (mqttspec re-exports the websocketspec constant), `HookContext.SetTx`, per-handler `runInTx`/`sendTxError`. Per message: read = 1 tx (Before/After hooks + queries); delete = 1 tx (Before, delete, After); create/update = tx 1 (Before + write) then tx 2 (re-fetch + `BeforeScan` + After). `create()`/`update()` no longer re-fetch; `read*`/`create`/`update`/`delete` use `hookCtx.Tx`. websocketspec `FetchRowNumber` keeps its public signature and delegates to a new tx-aware `fetchRowNumber`. A failure in begin/`OnTxBegin`/commit answers `transaction_error` with no detail. Tests: `pkg/websocketspec/tx_test.go` (sqlmock), `pkg/mqttspec/tx_test.go` (sqlite); mqttspec `update` tests now pass `Tx`.
- DECIDED in P4 (follow `AfterRead` question above): websocketspec/mqttspec run `AfterRead` inside the read tx (keeps "read has no second tx").
- DONE P5: resolvemcp. `OnTxBegin`, `HookContext.SetTx`, `Handler.runInTx`. Read = 1 tx (`BeforeRead`, count, scan, `AfterRead`; `readInTx`). Delete = 1 tx (`BeforeDelete` moved inside, after `OnTxBegin`). Create (single and batch, unified) = tx 1 (`BeforeCreate` + inserts) then tx 2 (re-fetch + `AfterCreate`); the old single-record pool insert/re-fetch is gone. Update = tx 1 (select, `BeforeUpdate`, update, `AfterUpdate`) then tx 2 (re-fetch). `BeforeHandle` still runs before any tx with `Tx = h.db`. Tests: `pkg/resolvemcp/tx_test.go` (sqlmock).
- DONE P6: funcspec. `OnTxBegin`, `HookContext.SetTx`, `Handler.runInTx` for `SqlQuery` and `SqlQueryList`. `BeforeResponse` now runs in a second short tx (`Tx` is no longer the pool). `BeforeOp` is unchanged (still per statement). A begin/`OnTxBegin`/commit failure answers 500 `transaction_error` / "Transaction failed" (before, it returned with no response); body failures still answer via `sendError`. Tests: `pkg/funcspec/tx_test.go`.
- DONE (AfterRead, decided by user): restheadspec `AfterRead` now runs in a second short tx. Test: `pkg/restheadspec/read_tx_test.go`.
- DONE P7: `pkg/security/txsettings.go`: `SecurityList.SetTxSettings(fn)`, `StampTxSettings`, `ApplyTxSettings` (configurable map, decided by user; `set_config(name, value, true)`, value hex-encoded, name validated, Postgres only, fail closed). Every spec's `RegisterSecurityHooks` registers it on `OnTxBegin`. Tests: `pkg/security/txsettings_test.go`, `pkg/resolvespec/tx_settings_test.go`. Docs: `pkg/common/TRANSACTIONS.md`.
- DONE real-Postgres check (resolvespec, testserver via compose): create `tx=1 pooled=0`, read `tx=1 pooled=0`, update `tx=2 pooled=0` (was `pooled=1`), single delete `tx=1 pooled=0`, batch create/delete `tx=1 pooled=0`. Compose now uses host networking (bridge fails here): testserver on 8123, Postgres on 8124 (was 8080/5434); integration test DSNs updated. Smoke script covers read and update. websocketspec/mqttspec/resolvemcp/restheadspec/funcspec not measured on real Postgres.
- DONE regression tests: per-spec read/create/update/delete hook-on-tx, failure-rollback (Before*/After*/`OnTxBegin`) and second-tx tests in all six specs (`ops_tx_test.go`, `tx_test.go`, `read_tx_test.go`); stamping tests for resolvespec, resolvemcp, funcspec; pgsql adapter preload tests (same connection, error returned); source guard `pkg/common/tx_guard_test.go` (no direct `RunInTransaction`/`BeginTx`, no `Tx = h.db` beyond the allowlisted BeforeHandle placeholders, no pool statements in spec handlers).
- FIXED: resolvespec now fires `AfterRead` (in the read tx; `Result` = scanned slice for single and list) `AfterCreate` (in the create tx, per record, all four create paths) and `AfterDelete` (in the delete tx, once per request; a failure rolls the delete back). Before, neither fired, so `AfterRead` column-level security masking (registered by `RegisterSecurityHooks`) was silently skipped on resolvespec reads. Failing `AfterRead`/`AfterCreate` fails the request and rolls back. Tests: `pkg/resolvespec/ops_tx_test.go` (incl. end-to-end column hiding).
- FIXED: restheadspec total-count cache key now includes the record id; a read by id (total 1) no longer poisons the list total for the 2-minute TTL. Tests: `pkg/restheadspec/cache_key_test.go`. resolvespec is unaffected (its count runs before the id filter, so the total is the list total by design). The cache is still process-wide, so read tests call `resetTotalCache`.
- NOTE: `pkg/security` `TestDatabaseAuthenticator` fails with `-count=2` (also on `cd96404`, before this work); use `-count=1`.
## Tests
- Existing: per-spec `handler_test.go`, `hooks_test.go`, `integration_test.go`; models in `pkg/testmodels/business.go`; `dbtrace` unit tests.
- Done: delete tx tests (`pkg/*/delete_tx_test.go`, sqlmock, 1-conn pool detects pool use). Missing: same for read/create/update, `OnTxBegin`, other specs.
- Add per spec/op: hook `Tx` is not the pool; `OnTxBegin` fires once per tx, before other hooks; single-ID delete = 1 tx; `dbtrace` `pooled == 0` on the request path.
- Test data: reuse `pkg/testmodels`; **ask before generating new data** (per project rule).
- Regression: full `go test -race` for security, dbmanager, common, restheadspec, resolvespec, websocketspec, mqttspec, resolvemcp, funcspec. Known pre-existing failures: mqttspec integration (no DB).
## Risks
- Long tx if a hook does slow work inside it → hold connection longer; keep hooks fast.
- Pool of 1: nothing inside a tx may take a second pool connection (auth/security loads are outside; keep it so).
- Behavior change: After hooks no longer get the pool handle; hooks that relied on an independent connection break.
- websocket/mqtt long-lived connections: tx must be per message, never per connection.
## Done when
- `dbtrace` shows `pooled=0` for every handler op on a hooked model.
- RLS GUC set in `OnTxBegin` is visible to read, create, update, delete queries and hooks.
- No `Tx: h.db` / `hookCtx.Tx = h.db` left in spec handlers.
- OPEN: websocketspec `BeforeDisconnect`/`AfterDisconnect` are defined but never executed (connection lifecycle, not DB). Allowlisted in `TestEveryDefinedHookHasACallSite`; wire them to remove the entry.
- DONE: column-level hide/mask columns are dropped from create/update payloads (`security.ApplyWriteColumnSecurity`); rules preloaded in `BeforeHandle` for create/update. resolvemcp update now runs `BeforeHandle`.
+12
View File
@@ -0,0 +1,12 @@
# Clients
| Dir | Language | Specs | Verified |
|---|---|---|---|
| `resolvespec-js` | TypeScript | ResolveSpec, HeaderSpec, FunctionSpec, WebSocketSpec | yes |
| `resolvespec-python` | Python >= 3.11 | ResolveSpec, HeaderSpec, FunctionSpec, WebSocketSpec | yes (61 tests) |
| `resolvespec-go` | Go | ResolveSpec, FunctionSpec | yes (`go test`) |
| `resolvespec-rs` | Rust | ResolveSpec, FunctionSpec | yes (`cargo test`) |
| `resolvespec-cs` | C# (.NET 8) | ResolveSpec, FunctionSpec | yes (`dotnet test`) |
| `resolvespec-dart` | Dart / Flutter | ResolveSpec, FunctionSpec | yes (`dart test`) |
Wire behaviour is identical across clients; FunctionSpec server quirks are listed in each README.
+2
View File
@@ -0,0 +1,2 @@
bin/
obj/
+43
View File
@@ -0,0 +1,43 @@
# ResolveSpec.Client (C#)
.NET 8 client for ResolveSpec (JSON body) and FunctionSpec. `System.Text.Json`, no other dependencies.
> Tests run with `DOTNET_ROLL_FORWARD=Major` when only a newer runtime than 8.0 is installed.
## Clients
| Type | Constructor | Methods |
|---|---|---|
| `ResolveSpecClient` | `(baseUrl, ClientOptions?)` | `GetMetadataAsync` `ReadAsync` `CreateAsync` `UpdateAsync` `DeleteAsync` |
| `FuncSpecClient` | `(baseUrl, ClientOptions?)` | `QueryAsync` `QueryListAsync` |
`ClientOptions`: `Token`, `Headers`, `Timeout`, `HttpClient`. Precedence: Content-Type < custom headers < bearer token.
## ResolveSpec
- `id`: int/long/string → URL, `IEnumerable<string>` → body.
- `Options` with nullable properties; wire names via `JsonPropertyName`.
- Result: `Response{Success, Data (JsonElement), Metadata}`; `resp.Decode<T>()`.
## FunctionSpec
- Routes are server-defined: pass the `path`.
- Params (`IDictionary<string, object?>`) → query string (enumerable → repeated keys, bool → `true`/`false`, null skipped).
- `FuncSpecOptions` → `X-*` headers: `Filters`, `SearchFilters`, `CustomSqlWhere`, `CustomSqlOr`, `Sort`, `Limit`, `Offset`, `Distinct`, `SkipCount`, `SkipCache`, `ResponseFormat`.
- `QueryListAsync` fills `Metadata` from `Content-Range`; 206 is success.
- Static helpers: `BuildHeaders`, `BuildQuery`, `EncodeHeaderValue`, `DecodeHeaderValue`.
## Server quirks
- `Sort` is raw SQL in ORDER BY (client sends `col ASC|DESC`).
- One search operator per column.
- Values starting `ZIP_` / `__` are base64-decoded by the server.
- Non-ASCII, control chars and edge spaces are auto-encoded (`ZIP_`).
## Errors
`ResolveSpecException{StatusCode, Message, Error{Code, Detail, Sql}}`.
## Test
`dotnet test tests/`
@@ -0,0 +1,187 @@
using System.Globalization;
using System.Text;
using System.Text.Json;
using System.Text.RegularExpressions;
namespace ResolveSpec;
/// <summary>
/// Options sent to funcspec endpoints as X-* headers.
/// Server behaviour (pkg/funcspec): Sort is inserted raw into ORDER BY (so it is sent as SQL terms);
/// only one search operator per column is kept; values starting with "ZIP_" or "__" are
/// base64-decoded by the server, so such plaintext values cannot be sent faithfully.
/// </summary>
public sealed class FuncSpecOptions
{
/// <summary>eq+AND -> X-FieldFilter; others X-SearchOp / X-SearchOr.</summary>
public List<FilterOption>? Filters { get; set; }
/// <summary>X-SearchFilter-{col}: text ILIKE.</summary>
public Dictionary<string, string>? SearchFilters { get; set; }
public string? CustomSqlWhere { get; set; }
public string? CustomSqlOr { get; set; }
public List<SortOption>? Sort { get; set; }
public int? Limit { get; set; }
public int? Offset { get; set; }
public bool? Distinct { get; set; }
public bool? SkipCount { get; set; }
public bool? SkipCache { get; set; }
/// <summary>simple | detail | syncfusion</summary>
public string? ResponseFormat { get; set; }
}
/// <summary>Client for user-defined SQL endpoints. Routes are defined by the server application.</summary>
public sealed class FuncSpecClient
{
readonly Transport _t;
public FuncSpecClient(string baseUrl, ClientOptions? options = null) => _t = new Transport(baseUrl, options);
static readonly Dictionary<string, string> OperatorMap = new()
{
["eq"] = "equals", ["neq"] = "notequals", ["gt"] = "greaterthan", ["gte"] = "greaterthanorequal",
["lt"] = "lessthan", ["lte"] = "lessthanorequal", ["like"] = "contains", ["ilike"] = "contains",
["contains"] = "contains", ["startswith"] = "beginswith", ["endswith"] = "endswith", ["in"] = "in",
["between"] = "between", ["between_inclusive"] = "betweeninclusive",
["is_null"] = "empty", ["is_not_null"] = "notempty",
};
static string Scalar(object? v) => v switch
{
null => "",
string s => s,
bool b => b ? "true" : "false",
JsonElement { ValueKind: JsonValueKind.Null } => "",
JsonElement e => e.ValueKind == JsonValueKind.String ? e.GetString() ?? "" : e.ToString(),
IFormattable f => f.ToString(null, CultureInfo.InvariantCulture),
_ => v.ToString() ?? "",
};
static string FilterValue(object? v) =>
v is System.Collections.IEnumerable list and not string
? string.Join(",", list.Cast<object?>().Select(Scalar))
: Scalar(v);
/// <summary>Base64 (UTF-8) with the ZIP_ prefix.</summary>
public static string EncodeHeaderValue(string v) => "ZIP_" + Convert.ToBase64String(Encoding.UTF8.GetBytes(v));
/// <summary>Decode a value that may carry a ZIP_ or __ prefix (nested allowed).</summary>
public static string DecodeHeaderValue(string v)
{
foreach (var p in new[] { "ZIP_", "__" })
{
if (!v.StartsWith(p, StringComparison.Ordinal)) continue;
var b64 = Regex.Replace(v[p.Length..], "[\n\r ]", "");
b64 = b64.PadRight(b64.Length + (4 - b64.Length % 4) % 4, '=');
try { return DecodeHeaderValue(Encoding.UTF8.GetString(Convert.FromBase64String(b64))); }
catch (FormatException) { return v; }
}
return v;
}
/// <summary>Encode values that are unsafe as raw header/query text (non-ASCII, control chars, edge spaces).</summary>
static string Safe(string v) =>
v != v.Trim() || v.Any(c => c > 127 || char.IsControl(c)) ? EncodeHeaderValue(v) : v;
/// <summary>Build the X-* headers understood by funcspec.ParseParameters.</summary>
public static Dictionary<string, string> BuildHeaders(FuncSpecOptions? o)
{
var h = new Dictionary<string, string>();
if (o == null) return h;
foreach (var f in o.Filters ?? new())
{
var logic = string.IsNullOrEmpty(f.LogicOperator) ? "AND" : f.LogicOperator;
var v = Safe(FilterValue(f.Value));
if (f.Operator == "eq" && logic == "AND") { h[$"X-FieldFilter-{f.Column}"] = v; continue; }
var op = OperatorMap.TryGetValue(f.Operator, out var m) ? m : f.Operator;
h[$"{(logic == "OR" ? "X-SearchOr" : "X-SearchOp")}-{op}-{f.Column}"] = v;
}
foreach (var (col, text) in o.SearchFilters ?? new()) h[$"X-SearchFilter-{col}"] = Safe(text);
if (!string.IsNullOrEmpty(o.CustomSqlWhere)) h["X-Custom-SQL-W"] = Safe(o.CustomSqlWhere);
if (!string.IsNullOrEmpty(o.CustomSqlOr)) h["X-Custom-SQL-Or"] = Safe(o.CustomSqlOr);
if (o.Sort is { Count: > 0 })
{
// funcspec puts this verbatim into ORDER BY
h["X-Sort"] = Safe(string.Join(",", o.Sort.Select(s =>
$"{s.Column} {(string.Equals(s.Direction, "desc", StringComparison.OrdinalIgnoreCase) ? "DESC" : "ASC")}")));
}
if (o.Limit != null) h["X-Limit"] = o.Limit.Value.ToString(CultureInfo.InvariantCulture);
if (o.Offset != null) h["X-Offset"] = o.Offset.Value.ToString(CultureInfo.InvariantCulture);
if (o.Distinct != null) h["X-Distinct"] = Bool(o.Distinct.Value);
if (o.SkipCount != null) h["X-SkipCount"] = Bool(o.SkipCount.Value);
if (o.SkipCache != null) h["X-SkipCache"] = Bool(o.SkipCache.Value);
switch (o.ResponseFormat)
{
case "simple": h["X-SimpleApi"] = "true"; break;
case "detail": h["X-DetailApi"] = "true"; break;
case "syncfusion": h["X-Syncfusion"] = "true"; break;
}
return h;
}
static string Bool(bool b) => b ? "true" : "false";
/// <summary>Build query-string pairs: bools -> true/false, lists -> repeated keys, null skipped.</summary>
public static List<KeyValuePair<string, string>> BuildQuery(IDictionary<string, object?>? p)
{
var o = new List<KeyValuePair<string, string>>();
foreach (var (k, v) in p ?? new Dictionary<string, object?>())
{
if (v == null) continue;
if (v is System.Collections.IEnumerable list and not string)
foreach (var e in list) o.Add(new(k, Safe(Scalar(e))));
else o.Add(new(k, Safe(Scalar(v))));
}
return o;
}
static readonly Regex ContentRange = new(@"(\d+)-(\d+)/(\d+)");
static Metadata MetadataFrom(string? contentRange, FuncSpecOptions? o)
{
var m = new Metadata { Limit = o?.Limit ?? 0 };
var g = ContentRange.Match(contentRange ?? "");
if (g.Success)
{
var start = long.Parse(g.Groups[1].Value, CultureInfo.InvariantCulture);
var end = long.Parse(g.Groups[2].Value, CultureInfo.InvariantCulture);
var total = long.Parse(g.Groups[3].Value, CultureInfo.InvariantCulture);
m.Total = total; m.Filtered = total; m.Count = end - start; m.Offset = start;
}
return m;
}
async Task<Response> CallAsync(HttpMethod method, string path, IDictionary<string, object?>? p, FuncSpecOptions? o, bool list, CancellationToken ct)
{
var url = $"{_t.BaseUrl}/{path.TrimStart('/')}";
var q = BuildQuery(p);
if (q.Count > 0)
url += "?" + string.Join("&", q.Select(kv => $"{Uri.EscapeDataString(kv.Key)}={Uri.EscapeDataString(kv.Value)}"));
var (resp, text) = await _t.SendAsync(method, url, null, BuildHeaders(o), ct).ConfigureAwait(false);
var status = (int)resp.StatusCode;
if (!resp.IsSuccessStatusCode) throw Transport.ErrorFrom(status, text, resp.ReasonPhrase); // 206 is success
var r = new Response
{
Success = true,
Data = string.IsNullOrWhiteSpace(text) ? JsonDocument.Parse("null").RootElement.Clone() : JsonDocument.Parse(text).RootElement.Clone(),
};
if (list)
{
// Content-Range is a content header in HttpClient; fall back to response headers.
IEnumerable<string>? cr = null;
if (!resp.Content.Headers.TryGetValues("Content-Range", out cr)) resp.Headers.TryGetValues("Content-Range", out cr);
r.Metadata = MetadataFrom(cr?.FirstOrDefault(), o);
}
return r;
}
/// <summary>Single-record endpoint (SqlQuery). Data is the row object.</summary>
public Task<Response> QueryAsync(string path, IDictionary<string, object?>? p = null, FuncSpecOptions? o = null, HttpMethod? method = null, CancellationToken ct = default) =>
CallAsync(method ?? HttpMethod.Get, path, p, o, false, ct);
/// <summary>List endpoint (SqlQueryList). Metadata comes from Content-Range.</summary>
public Task<Response> QueryListAsync(string path, IDictionary<string, object?>? p = null, FuncSpecOptions? o = null, HttpMethod? method = null, CancellationToken ct = default) =>
CallAsync(method ?? HttpMethod.Get, path, p, o, true, ct);
}
+78
View File
@@ -0,0 +1,78 @@
using System.Net.Http.Headers;
using System.Text;
using System.Text.Json;
namespace ResolveSpec;
/// <summary>Shared HTTP configuration for both clients.</summary>
public sealed class ClientOptions
{
public string? Token { get; set; }
public Dictionary<string, string> Headers { get; } = new(StringComparer.OrdinalIgnoreCase);
public TimeSpan Timeout { get; set; } = TimeSpan.FromSeconds(30);
/// <summary>Supply your own HttpClient (tests, pooling). Its BaseAddress is ignored.</summary>
public HttpClient? HttpClient { get; set; }
}
internal sealed class Transport
{
public readonly string BaseUrl;
readonly ClientOptions _o;
readonly HttpClient _http;
public Transport(string baseUrl, ClientOptions? o)
{
BaseUrl = baseUrl.TrimEnd('/');
_o = o ?? new ClientOptions();
_http = _o.HttpClient ?? new HttpClient { Timeout = _o.Timeout };
}
/// <summary>Content-Type &lt; custom headers &lt; per-call headers &lt; bearer token.</summary>
public async Task<(HttpResponseMessage resp, string body)> SendAsync(
HttpMethod method, string url, string? json, IDictionary<string, string>? extra, CancellationToken ct)
{
using var req = new HttpRequestMessage(method, url);
if (json != null) req.Content = new StringContent(json, Encoding.UTF8, "application/json");
foreach (var (k, v) in _o.Headers) Set(req, k, v);
if (extra != null) foreach (var (k, v) in extra) Set(req, k, v);
if (!string.IsNullOrEmpty(_o.Token)) req.Headers.Authorization = new AuthenticationHeaderValue("Bearer", _o.Token);
var resp = await _http.SendAsync(req, ct).ConfigureAwait(false);
var body = await resp.Content.ReadAsStringAsync(ct).ConfigureAwait(false);
return (resp, body);
}
static void Set(HttpRequestMessage req, string name, string value)
{
req.Headers.Remove(name);
if (!req.Headers.TryAddWithoutValidation(name, value) && req.Content != null)
{
req.Content.Headers.Remove(name);
req.Content.Headers.TryAddWithoutValidation(name, value);
}
}
public static ResolveSpecException ErrorFrom(int status, string body, string? reason)
{
ApiError? err = null;
var isJson = false;
try
{
using var doc = JsonDocument.Parse(body);
isJson = true;
if (doc.RootElement.ValueKind == JsonValueKind.Object && doc.RootElement.TryGetProperty("error", out var e) && e.ValueKind == JsonValueKind.Object)
err = e.Deserialize<ApiError>();
}
catch (JsonException) { }
var message = err?.Message;
if (string.IsNullOrEmpty(message))
{
var text = isJson ? "" : body.Trim();
if (text.Length > 200) text = text[..200];
message = text.Length > 0 ? text : $"{reason ?? "Error"} ({status})";
}
return new ResolveSpecException(message, status, err);
}
public static string Segment(string s) => Uri.EscapeDataString(s);
}
@@ -0,0 +1,11 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net8.0</TargetFramework>
<Nullable>enable</Nullable>
<ImplicitUsings>enable</ImplicitUsings>
<RootNamespace>ResolveSpec</RootNamespace>
<PackageId>ResolveSpec.Client</PackageId>
<Version>0.1.0</Version>
<Description>Client for ResolveSpec (JSON body) and FunctionSpec endpoints</Description>
</PropertyGroup>
</Project>
@@ -0,0 +1,65 @@
using System.Text.Json;
using System.Text.Json.Serialization;
namespace ResolveSpec;
/// <summary>Client for the ResolveSpec JSON body protocol: POST {operation, data, options}.</summary>
public sealed class ResolveSpecClient
{
static readonly JsonSerializerOptions Json = new() { DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull };
readonly Transport _t;
public ResolveSpecClient(string baseUrl, ClientOptions? options = null) => _t = new Transport(baseUrl, options);
sealed class Request
{
[JsonPropertyName("operation")] public string Operation { get; set; } = "";
[JsonPropertyName("id")] public string[]? Id { get; set; }
[JsonPropertyName("data")] public object? Data { get; set; }
[JsonPropertyName("options")] public Options? Options { get; set; }
}
// A single id (int/long/string) goes in the URL; string[] / IEnumerable<string> goes in the body.
static string? UrlId(object? id) => id switch
{
null => null,
string s => s,
IEnumerable<string> => null,
_ => Convert.ToString(id, System.Globalization.CultureInfo.InvariantCulture),
};
static string[]? BodyId(object? id) => id is IEnumerable<string> e ? e.ToArray() : null;
string Url(string schema, string entity, string? id)
{
var u = $"{_t.BaseUrl}/{Transport.Segment(schema)}/{Transport.Segment(entity)}";
return string.IsNullOrEmpty(id) ? u : $"{u}/{Transport.Segment(id)}";
}
async Task<Response> SendAsync(HttpMethod method, string url, Request? body, CancellationToken ct)
{
var json = body == null ? null : JsonSerializer.Serialize(body, Json);
var (resp, text) = await _t.SendAsync(method, url, json, null, ct).ConfigureAwait(false);
var status = (int)resp.StatusCode;
if (!resp.IsSuccessStatusCode) throw Transport.ErrorFrom(status, text, resp.ReasonPhrase);
var r = JsonSerializer.Deserialize<Response>(text, Json) ?? new Response();
if (!r.Success && r.Error != null) throw new ResolveSpecException(r.Error.Message, status, r.Error);
return r;
}
/// <summary>GET /{schema}/{entity}</summary>
public Task<Response> GetMetadataAsync(string schema, string entity, CancellationToken ct = default) =>
SendAsync(HttpMethod.Get, Url(schema, entity, null), null, ct);
public Task<Response> ReadAsync(string schema, string entity, object? id = null, Options? options = null, CancellationToken ct = default) =>
SendAsync(HttpMethod.Post, Url(schema, entity, UrlId(id)), new Request { Operation = "read", Id = BodyId(id), Options = options }, ct);
public Task<Response> CreateAsync(string schema, string entity, object data, Options? options = null, CancellationToken ct = default) =>
SendAsync(HttpMethod.Post, Url(schema, entity, null), new Request { Operation = "create", Data = data, Options = options }, ct);
public Task<Response> UpdateAsync(string schema, string entity, object data, object? id = null, Options? options = null, CancellationToken ct = default) =>
SendAsync(HttpMethod.Post, Url(schema, entity, UrlId(id)), new Request { Operation = "update", Id = BodyId(id), Data = data, Options = options }, ct);
public Task<Response> DeleteAsync(string schema, string entity, object id, CancellationToken ct = default) =>
SendAsync(HttpMethod.Post, Url(schema, entity, UrlId(id)), new Request { Operation = "delete" }, ct);
}
+137
View File
@@ -0,0 +1,137 @@
using System.Text.Json;
using System.Text.Json.Serialization;
namespace ResolveSpec;
// Types aligned with Go pkg/common/types.go. JsonPropertyName values are the wire names.
public sealed class FilterOption
{
[JsonPropertyName("column")] public string Column { get; set; } = "";
/// <summary>eq neq gt gte lt lte like ilike in contains startswith endswith between between_inclusive is_null is_not_null</summary>
[JsonPropertyName("operator")] public string Operator { get; set; } = "eq";
[JsonPropertyName("value")] public object? Value { get; set; }
/// <summary>AND | OR</summary>
[JsonPropertyName("logic_operator")] public string? LogicOperator { get; set; }
}
public sealed class SortOption
{
[JsonPropertyName("column")] public string Column { get; set; } = "";
/// <summary>asc | desc</summary>
[JsonPropertyName("direction")] public string Direction { get; set; } = "asc";
}
public sealed class Parameter
{
[JsonPropertyName("name")] public string Name { get; set; } = "";
[JsonPropertyName("value")] public string Value { get; set; } = "";
[JsonPropertyName("sequence")] public int? Sequence { get; set; }
}
public sealed class CustomOperator
{
[JsonPropertyName("name")] public string Name { get; set; } = "";
[JsonPropertyName("sql")] public string Sql { get; set; } = "";
}
public sealed class ComputedColumn
{
[JsonPropertyName("name")] public string Name { get; set; } = "";
[JsonPropertyName("expression")] public string Expression { get; set; } = "";
}
public sealed class PreloadOption
{
[JsonPropertyName("relation")] public string? Relation { get; set; }
[JsonPropertyName("table_name")] public string? TableName { get; set; }
[JsonPropertyName("columns")] public List<string>? Columns { get; set; }
[JsonPropertyName("omit_columns")] public List<string>? OmitColumns { get; set; }
[JsonPropertyName("sort")] public List<SortOption>? Sort { get; set; }
[JsonPropertyName("filters")] public List<FilterOption>? Filters { get; set; }
[JsonPropertyName("where")] public string? Where { get; set; }
[JsonPropertyName("limit")] public int? Limit { get; set; }
[JsonPropertyName("offset")] public int? Offset { get; set; }
[JsonPropertyName("updateable")] public bool? Updateable { get; set; }
[JsonPropertyName("computed_ql")] public Dictionary<string, string>? ComputedQl { get; set; }
[JsonPropertyName("recursive")] public bool? Recursive { get; set; }
[JsonPropertyName("primary_key")] public string? PrimaryKey { get; set; }
[JsonPropertyName("related_key")] public string? RelatedKey { get; set; }
[JsonPropertyName("foreign_key")] public string? ForeignKey { get; set; }
[JsonPropertyName("recursive_child_key")] public string? RecursiveChildKey { get; set; }
[JsonPropertyName("sql_joins")] public List<string>? SqlJoins { get; set; }
[JsonPropertyName("join_aliases")] public List<string>? JoinAliases { get; set; }
}
public sealed class VectorSearchOption
{
[JsonPropertyName("column")] public string Column { get; set; } = "";
[JsonPropertyName("vector")] public List<double> Vector { get; set; } = new();
/// <summary>l2 (default) | cosine | ip</summary>
[JsonPropertyName("metric")] public string? Metric { get; set; }
/// <summary>Distance column alias, default _distance.</summary>
[JsonPropertyName("as")] public string? As { get; set; }
[JsonPropertyName("direction")] public string? Direction { get; set; }
}
/// <summary>ResolveSpec request options object.</summary>
public sealed class Options
{
[JsonPropertyName("preload")] public List<PreloadOption>? Preload { get; set; }
[JsonPropertyName("columns")] public List<string>? Columns { get; set; }
[JsonPropertyName("omit_columns")] public List<string>? OmitColumns { get; set; }
[JsonPropertyName("filters")] public List<FilterOption>? Filters { get; set; }
[JsonPropertyName("sort")] public List<SortOption>? Sort { get; set; }
[JsonPropertyName("limit")] public int? Limit { get; set; }
[JsonPropertyName("offset")] public int? Offset { get; set; }
[JsonPropertyName("customOperators")] public List<CustomOperator>? CustomOperators { get; set; }
[JsonPropertyName("computedColumns")] public List<ComputedColumn>? ComputedColumns { get; set; }
[JsonPropertyName("parameters")] public List<Parameter>? Parameters { get; set; }
[JsonPropertyName("cursor_forward")] public string? CursorForward { get; set; }
[JsonPropertyName("cursor_backward")] public string? CursorBackward { get; set; }
[JsonPropertyName("fetch_row_number")] public string? FetchRowNumber { get; set; }
[JsonPropertyName("vector_search")] public VectorSearchOption? VectorSearch { get; set; }
}
public sealed class Metadata
{
[JsonPropertyName("total")] public long Total { get; set; }
[JsonPropertyName("count")] public long Count { get; set; }
[JsonPropertyName("filtered")] public long Filtered { get; set; }
[JsonPropertyName("limit")] public long Limit { get; set; }
[JsonPropertyName("offset")] public long Offset { get; set; }
}
public sealed class ApiError
{
[JsonPropertyName("code")] public string Code { get; set; } = "";
[JsonPropertyName("message")] public string Message { get; set; } = "";
[JsonPropertyName("details")] public JsonElement? Details { get; set; }
/// <summary>Server-side reason (funcspec / restheadspec).</summary>
[JsonPropertyName("detail")] public string? Detail { get; set; }
[JsonPropertyName("sql")] public string? Sql { get; set; }
}
/// <summary>ResolveSpec envelope. <see cref="Data"/> is raw JSON; use <see cref="Decode{T}"/>.</summary>
public sealed class Response
{
[JsonPropertyName("success")] public bool Success { get; set; }
[JsonPropertyName("data")] public JsonElement Data { get; set; }
[JsonPropertyName("metadata")] public Metadata? Metadata { get; set; }
[JsonPropertyName("error")] public ApiError? Error { get; set; }
public T? Decode<T>() => Data.ValueKind == JsonValueKind.Undefined ? default : Data.Deserialize<T>();
}
/// <summary>Thrown on a non-2xx response or an unsuccessful API result.</summary>
public sealed class ResolveSpecException : Exception
{
public int StatusCode { get; }
public ApiError Error { get; }
public ResolveSpecException(string message, int statusCode, ApiError? error = null) : base(message)
{
StatusCode = statusCode;
Error = error ?? new ApiError { Message = message };
}
}
+179
View File
@@ -0,0 +1,179 @@
using System.Net;
using System.Text;
using System.Text.Json;
using ResolveSpec;
using Xunit;
public class Stub : HttpMessageHandler
{
public HttpRequestMessage? Request;
public string Body = "";
readonly HttpStatusCode _status;
readonly string _json;
readonly Dictionary<string, string> _headers;
public Stub(HttpStatusCode status, string json, Dictionary<string, string>? headers = null)
{
_status = status; _json = json; _headers = headers ?? new();
}
protected override async Task<HttpResponseMessage> SendAsync(HttpRequestMessage request, CancellationToken ct)
{
Request = request;
Body = request.Content == null ? "" : await request.Content.ReadAsStringAsync(ct);
var r = new HttpResponseMessage(_status) { Content = new StringContent(_json, Encoding.UTF8, "application/json") };
foreach (var (k, v) in _headers)
if (!r.Headers.TryAddWithoutValidation(k, v)) r.Content.Headers.TryAddWithoutValidation(k, v);
return r;
}
}
public class ResolveSpecTests
{
static (ResolveSpecClient, Stub) Make(HttpStatusCode s, string json)
{
var stub = new Stub(s, json);
var o = new ClientOptions { Token = "tok", HttpClient = new HttpClient(stub) };
o.Headers["X-Tenant"] = "a";
return (new ResolveSpecClient("http://localhost:3000/", o), stub);
}
[Fact]
public async Task ReadPostsBody()
{
var (c, s) = Make(HttpStatusCode.OK, """{"success":true,"data":[{"id":1}]}""");
var r = await c.ReadAsync("public", "users", null, new Options { Limit = 5, Filters = new() { new FilterOption { Column = "a", Operator = "eq", Value = 1 } } });
Assert.Equal(HttpMethod.Post, s.Request!.Method);
Assert.Equal("/public/users", s.Request.RequestUri!.AbsolutePath);
Assert.Equal("Bearer tok", s.Request.Headers.Authorization!.ToString());
Assert.Equal("a", s.Request.Headers.GetValues("X-Tenant").Single());
using var body = JsonDocument.Parse(s.Body);
Assert.Equal("read", body.RootElement.GetProperty("operation").GetString());
Assert.Equal(5, body.RootElement.GetProperty("options").GetProperty("limit").GetInt32());
Assert.False(body.RootElement.TryGetProperty("id", out _));
Assert.Single(r.Decode<List<Dictionary<string, int>>>()!);
}
[Fact]
public async Task IdPlacement()
{
var (c, s) = Make(HttpStatusCode.OK, """{"success":true,"data":{}}""");
await c.ReadAsync("s", "e", 7);
Assert.Equal("/s/e/7", s.Request!.RequestUri!.AbsolutePath);
await c.UpdateAsync("s", "e", new { a = 1 }, new[] { "1", "2" });
Assert.Equal("/s/e", s.Request!.RequestUri!.AbsolutePath);
using (var b = JsonDocument.Parse(s.Body))
{
Assert.Equal(2, b.RootElement.GetProperty("id").GetArrayLength());
Assert.Equal("update", b.RootElement.GetProperty("operation").GetString());
}
await c.DeleteAsync("s", "e", "a/b");
Assert.Equal("/s/e/a%2Fb", s.Request!.RequestUri!.AbsoluteUri[(s.Request.RequestUri.AbsoluteUri.IndexOf("/s/e", StringComparison.Ordinal))..]);
Assert.Contains("\"delete\"", s.Body);
}
[Fact]
public async Task Errors()
{
var (c, _) = Make(HttpStatusCode.BadRequest, """{"success":false,"error":{"code":"x","message":"bad","detail":"why"}}""");
var e = await Assert.ThrowsAsync<ResolveSpecException>(() => c.ReadAsync("s", "e"));
Assert.Equal((400, "x", "bad", "why"), (e.StatusCode, e.Error.Code, e.Message, e.Error.Detail));
var (c2, _) = Make(HttpStatusCode.BadGateway, "bad gateway");
var e2 = await Assert.ThrowsAsync<ResolveSpecException>(() => c2.ReadAsync("s", "e"));
Assert.Equal((502, "bad gateway"), (e2.StatusCode, e2.Message));
var (c3, _) = Make(HttpStatusCode.OK, """{"success":false,"error":{"code":"c","message":"nope"}}""");
var e3 = await Assert.ThrowsAsync<ResolveSpecException>(() => c3.ReadAsync("s", "e"));
Assert.Equal("nope", e3.Message);
}
}
public class FuncSpecTests
{
[Fact]
public void HeaderFilters()
{
var h = FuncSpecClient.BuildHeaders(new FuncSpecOptions
{
Filters = new()
{
new() { Column = "status", Operator = "eq", Value = "active" },
new() { Column = "age", Operator = "gte", Value = 18 },
new() { Column = "name", Operator = "contains", Value = "x", LogicOperator = "OR" },
new() { Column = "deleted", Operator = "is_null" },
new() { Column = "id", Operator = "in", Value = new[] { 1, 2 } },
new() { Column = "p", Operator = "between_inclusive", Value = new[] { 1, 5 } },
},
});
Assert.Equal(new Dictionary<string, string>
{
["X-FieldFilter-status"] = "active",
["X-SearchOp-greaterthanorequal-age"] = "18",
["X-SearchOr-contains-name"] = "x",
["X-SearchOp-empty-deleted"] = "",
["X-SearchOp-in-id"] = "1,2",
["X-SearchOp-betweeninclusive-p"] = "1,5",
}, h);
}
[Fact]
public void HeaderMiscAndEncoding()
{
var h = FuncSpecClient.BuildHeaders(new FuncSpecOptions
{
SearchFilters = new() { ["name"] = "bob" }, CustomSqlWhere = "a = 1", CustomSqlOr = "b = 2",
Sort = new() { new() { Column = "name", Direction = "asc" }, new() { Column = "created_at", Direction = "DESC" } },
Limit = 5, Offset = 10, Distinct = true, SkipCount = true, SkipCache = false, ResponseFormat = "syncfusion",
});
Assert.Equal("name ASC,created_at DESC", h["X-Sort"]);
Assert.Equal("bob", h["X-SearchFilter-name"]);
Assert.Equal("a = 1", h["X-Custom-SQL-W"]);
Assert.Equal("false", h["X-SkipCache"]);
Assert.Equal("true", h["X-Syncfusion"]);
h = FuncSpecClient.BuildHeaders(new FuncSpecOptions { Filters = new()
{
new() { Column = "n", Operator = "eq", Value = "héllo" },
new() { Column = "m", Operator = "eq", Value = " pad" },
} });
Assert.StartsWith("ZIP_", h["X-FieldFilter-n"]);
Assert.Equal("héllo", FuncSpecClient.DecodeHeaderValue(h["X-FieldFilter-n"]));
Assert.Equal(" pad", FuncSpecClient.DecodeHeaderValue(h["X-FieldFilter-m"]));
}
[Fact]
public void QueryBuilding()
{
var q = FuncSpecClient.BuildQuery(new Dictionary<string, object?> { ["a"] = true, ["b"] = new[] { "x", "y" }, ["c"] = null, ["d"] = 3 });
Assert.Equal(new[] { "a=true", "b=x", "b=y", "d=3" }, q.Select(kv => $"{kv.Key}={kv.Value}"));
}
[Fact]
public async Task QueryListMetadata()
{
var stub = new Stub((HttpStatusCode)206, """[{"id":1},{"id":2}]""", new() { ["Content-Range"] = "items 10-12/50" });
var c = new FuncSpecClient("http://x", new ClientOptions { Token = "tok", HttpClient = new HttpClient(stub) });
var r = await c.QueryListAsync("/api/users", new Dictionary<string, object?> { ["org"] = 1 }, new FuncSpecOptions { Limit = 2 });
Assert.Equal("GET", stub.Request!.Method.Method);
Assert.Equal("/api/users", stub.Request.RequestUri!.AbsolutePath);
Assert.Equal("?org=1", stub.Request.RequestUri.Query);
Assert.Equal("2", stub.Request.Headers.GetValues("X-Limit").Single());
Assert.Equal((50L, 2L, 50L, 2L, 10L), (r.Metadata!.Total, r.Metadata.Count, r.Metadata.Filtered, r.Metadata.Limit, r.Metadata.Offset));
Assert.Equal(2, r.Data.GetArrayLength());
}
[Fact]
public async Task QuerySingleAndError()
{
var ok = new FuncSpecClient("http://x", new ClientOptions { HttpClient = new HttpClient(new Stub(HttpStatusCode.OK, """{"id":1}""")) });
var r = await ok.QueryAsync("api/u");
Assert.Null(r.Metadata);
Assert.Equal(1, r.Data.GetProperty("id").GetInt32());
var bad = new FuncSpecClient("http://x", new ClientOptions { HttpClient = new HttpClient(new Stub(HttpStatusCode.BadRequest,
"""{"success":false,"error":{"code":"hook_error","message":"Hook execution failed","detail":"authentication required"}}""")) });
var e = await Assert.ThrowsAsync<ResolveSpecException>(() => bad.QueryAsync("api/u"));
Assert.Equal(("hook_error", "authentication required"), (e.Error.Code, e.Error.Detail));
}
}
@@ -0,0 +1,16 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net8.0</TargetFramework>
<Nullable>enable</Nullable>
<ImplicitUsings>enable</ImplicitUsings>
<IsPackable>false</IsPackable>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.11.1" />
<PackageReference Include="xunit" Version="2.9.2" />
<PackageReference Include="xunit.runner.visualstudio" Version="2.8.2" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="../src/ResolveSpec.csproj" />
</ItemGroup>
</Project>
+3
View File
@@ -0,0 +1,3 @@
.dart_tool/
pubspec.lock
build/
+41
View File
@@ -0,0 +1,41 @@
# resolvespec (Dart)
Dart / Flutter client for ResolveSpec (JSON body) and FunctionSpec. Depends on `package:http`. Dart >= 3.3.
## Clients
| Type | Constructor | Methods |
|---|---|---|
| `ResolveSpecClient` | `(baseUrl, [ClientOptions])` | `getMetadata` `read` `create` `update` `delete` `close` |
| `FuncSpecClient` | `(baseUrl, [ClientOptions])` | `query` `queryList` `close` |
`ClientOptions(token:, headers:, timeout:, httpClient:)`. Precedence: Content-Type < custom headers < bearer token.
## ResolveSpec
- `id`: `int`/`String` → URL, `List<String>` → body. Named args: `id:`, `options:`.
- `Options`, `FilterOption(column, operator, [value, logic])`, `SortOption(column, [direction])`.
- Result: `Response{success, data (decoded JSON), metadata}`.
## FunctionSpec
- Routes are server-defined: pass the `path`.
- `params:` map → query string (list → repeated keys, null skipped).
- `FuncSpecOptions` → `X-*` headers: `filters`, `searchFilters`, `customSqlWhere`, `customSqlOr`, `sort`, `limit`, `offset`, `distinct`, `skipCount`, `skipCache`, `responseFormat`.
- `queryList` fills `metadata` from `Content-Range`; 206 is success.
- Helpers: `buildHeaders`, `buildQuery`, `encodeHeaderValue`, `decodeHeaderValue`.
## Server quirks
- `sort` is raw SQL in ORDER BY (client sends `col ASC|DESC`).
- One search operator per column.
- Values starting `ZIP_` / `__` are base64-decoded by the server.
- Non-ASCII, control chars and edge spaces are auto-encoded (`ZIP_`).
## Errors
`ResolveSpecException{statusCode, message, error: ApiError{code, detail, sql}}`.
## Test
`dart test`
@@ -0,0 +1 @@
include: package:lints/recommended.yaml
@@ -0,0 +1,7 @@
/// Client for ResolveSpec (JSON body) and FunctionSpec endpoints.
library;
export 'src/client.dart' show ClientOptions, ResolveSpecException;
export 'src/funcspec.dart';
export 'src/resolvespec.dart';
export 'src/types.dart';
@@ -0,0 +1,98 @@
import 'dart:convert';
import 'package:http/http.dart' as http;
import 'types.dart';
/// Thrown on a non-2xx response or an unsuccessful API result.
class ResolveSpecException implements Exception {
final int statusCode;
final String message;
final ApiError error;
ResolveSpecException(this.message, this.statusCode, [ApiError? error])
: error = error ?? ApiError(message: message);
@override
String toString() => 'ResolveSpecException($statusCode): $message';
}
/// Shared HTTP configuration for both clients.
class ClientOptions {
final String? token;
final Map<String, String> headers;
final Duration timeout;
/// Supply your own client (tests, pooling).
final http.Client? httpClient;
const ClientOptions(
{this.token,
this.headers = const {},
this.timeout = const Duration(seconds: 30),
this.httpClient});
}
class Transport {
final String baseUrl;
final ClientOptions options;
final http.Client _http;
Transport(String baseUrl, ClientOptions? options)
: baseUrl = baseUrl.replaceAll(RegExp(r'/+$'), ''),
options = options ?? const ClientOptions(),
_http = options?.httpClient ?? http.Client();
/// Content-Type < custom headers < per-call headers < bearer token.
Future<http.Response> send(String method, Uri uri,
{String? body, Map<String, String>? extra}) {
final headers = <String, String>{'Content-Type': 'application/json'};
void merge(Map<String, String> src) {
for (final e in src.entries) {
headers.removeWhere((k, _) => k.toLowerCase() == e.key.toLowerCase());
headers[e.key] = e.value;
}
}
merge(options.headers);
if (extra != null) merge(extra);
final token = options.token;
if (token != null && token.isNotEmpty) {
merge({'Authorization': 'Bearer $token'});
}
final req = http.Request(method, uri)..headers.addAll(headers);
if (body != null) req.body = body;
return _http
.send(req)
.timeout(options.timeout)
.then(http.Response.fromStream);
}
void close() => _http.close();
static ResolveSpecException errorFrom(http.Response resp) {
final body = utf8.decode(resp.bodyBytes, allowMalformed: true);
ApiError? err;
var isJson = false;
try {
final parsed = jsonDecode(body);
isJson = true;
if (parsed is Map<String, dynamic> &&
parsed['error'] is Map<String, dynamic>) {
err = ApiError.fromJson(parsed['error'] as Map<String, dynamic>);
}
} on FormatException {
// not JSON
}
var message = err?.message ?? '';
if (message.isEmpty) {
var text = isJson ? '' : body.trim();
if (text.length > 200) text = text.substring(0, 200);
message = text.isNotEmpty
? text
: '${resp.reasonPhrase ?? 'Error'} (${resp.statusCode})';
}
return ResolveSpecException(message, resp.statusCode, err);
}
}
@@ -0,0 +1,223 @@
import 'dart:convert';
import 'client.dart';
import 'types.dart';
/// Options sent to funcspec endpoints as X-* headers.
///
/// Server behaviour (pkg/funcspec): [sort] is inserted raw into ORDER BY (so it is sent as SQL
/// terms); only one search operator per column is kept; values starting with `ZIP_` or `__`
/// are base64-decoded by the server, so such plaintext values cannot be sent faithfully.
class FuncSpecOptions {
/// eq+AND -> X-FieldFilter; others X-SearchOp / X-SearchOr.
final List<FilterOption>? filters;
/// X-SearchFilter-{col}: text ILIKE.
final Map<String, String>? searchFilters;
final String? customSqlWhere;
final String? customSqlOr;
final List<SortOption>? sort;
final int? limit;
final int? offset;
final bool? distinct;
final bool? skipCount;
final bool? skipCache;
/// simple | detail | syncfusion
final String? responseFormat;
const FuncSpecOptions({
this.filters,
this.searchFilters,
this.customSqlWhere,
this.customSqlOr,
this.sort,
this.limit,
this.offset,
this.distinct,
this.skipCount,
this.skipCache,
this.responseFormat,
});
}
const _operatorMap = {
'eq': 'equals',
'neq': 'notequals',
'gt': 'greaterthan',
'gte': 'greaterthanorequal',
'lt': 'lessthan',
'lte': 'lessthanorequal',
'like': 'contains',
'ilike': 'contains',
'contains': 'contains',
'startswith': 'beginswith',
'endswith': 'endswith',
'in': 'in',
'between': 'between',
'between_inclusive': 'betweeninclusive',
'is_null': 'empty',
'is_not_null': 'notempty',
};
String _scalar(Object? v) => v == null ? '' : v.toString();
String _filterValue(Object? v) =>
v is Iterable ? v.map(_scalar).join(',') : _scalar(v);
/// Base64 (UTF-8) with the `ZIP_` prefix.
String encodeHeaderValue(String v) => 'ZIP_${base64.encode(utf8.encode(v))}';
/// Decode a value that may carry a `ZIP_` or `__` prefix (nested allowed).
String decodeHeaderValue(String v) {
for (final p in const ['ZIP_', '__']) {
if (v.startsWith(p)) {
var b64 = v.substring(p.length).replaceAll(RegExp(r'[\n\r ]'), '');
b64 = b64.padRight(b64.length + (4 - b64.length % 4) % 4, '=');
try {
return decodeHeaderValue(utf8.decode(base64.decode(b64)));
} on FormatException {
return v;
}
}
}
return v;
}
/// Encode values that are unsafe as raw header/query text (non-ASCII, control chars, edge spaces).
String _safe(String v) {
final unsafe =
v != v.trim() || v.runes.any((c) => c > 127 || c < 32 || c == 127);
return unsafe ? encodeHeaderValue(v) : v;
}
/// Build the X-* headers understood by funcspec.ParseParameters.
Map<String, String> buildHeaders(FuncSpecOptions? o) {
final h = <String, String>{};
if (o == null) return h;
for (final f in o.filters ?? const <FilterOption>[]) {
final logic = f.logicOperator ?? 'AND';
final v = _safe(_filterValue(f.value));
if (f.operator == 'eq' && logic == 'AND') {
h['X-FieldFilter-${f.column}'] = v;
} else {
final kind = logic == 'OR' ? 'X-SearchOr' : 'X-SearchOp';
h['$kind-${_operatorMap[f.operator] ?? f.operator}-${f.column}'] = v;
}
}
o.searchFilters
?.forEach((col, text) => h['X-SearchFilter-$col'] = _safe(text));
if (o.customSqlWhere != null && o.customSqlWhere!.isNotEmpty) {
h['X-Custom-SQL-W'] = _safe(o.customSqlWhere!);
}
if (o.customSqlOr != null && o.customSqlOr!.isNotEmpty) {
h['X-Custom-SQL-Or'] = _safe(o.customSqlOr!);
}
if (o.sort != null && o.sort!.isNotEmpty) {
// funcspec puts this verbatim into ORDER BY
h['X-Sort'] = _safe(o.sort!
.map((s) =>
'${s.column} ${s.direction.toLowerCase() == 'desc' ? 'DESC' : 'ASC'}')
.join(','));
}
if (o.limit != null) h['X-Limit'] = '${o.limit}';
if (o.offset != null) h['X-Offset'] = '${o.offset}';
if (o.distinct != null) h['X-Distinct'] = '${o.distinct}';
if (o.skipCount != null) h['X-SkipCount'] = '${o.skipCount}';
if (o.skipCache != null) h['X-SkipCache'] = '${o.skipCache}';
switch (o.responseFormat) {
case 'simple':
h['X-SimpleApi'] = 'true';
case 'detail':
h['X-DetailApi'] = 'true';
case 'syncfusion':
h['X-Syncfusion'] = 'true';
}
return h;
}
/// Build query-string pairs: lists -> repeated keys, null skipped, bools -> true/false.
Map<String, List<String>> buildQuery(Map<String, Object?>? params) {
final out = <String, List<String>>{};
params?.forEach((k, v) {
if (v == null) return;
out[k] = v is Iterable
? v.map((e) => _safe(_scalar(e))).toList()
: [_safe(_scalar(v))];
});
return out;
}
String? _header(Map<String, String> headers, String name) {
for (final e in headers.entries) {
if (e.key.toLowerCase() == name) return e.value;
}
return null;
}
final _contentRange = RegExp(r'(\d+)-(\d+)/(\d+)');
Metadata _metadata(String? contentRange, FuncSpecOptions? o) {
final m = _contentRange.firstMatch(contentRange ?? '');
if (m == null) return Metadata(limit: o?.limit ?? 0);
final start = int.parse(m.group(1)!);
final end = int.parse(m.group(2)!);
final total = int.parse(m.group(3)!);
return Metadata(
total: total,
count: end - start,
filtered: total,
limit: o?.limit ?? 0,
offset: start);
}
/// Client for user-defined SQL endpoints. Routes are defined by the server application.
class FuncSpecClient {
final Transport _t;
FuncSpecClient(String baseUrl, [ClientOptions? options])
: _t = Transport(baseUrl, options);
void close() => _t.close();
Future<Response> _call(String method, String path,
Map<String, Object?>? params, FuncSpecOptions? o, bool list) async {
final base =
Uri.parse('${_t.baseUrl}/${path.replaceAll(RegExp(r'^/+'), '')}');
final pairs = <String>[];
buildQuery(params).forEach((k, vs) {
for (final v in vs) {
pairs.add(
'${Uri.encodeQueryComponent(k)}=${Uri.encodeQueryComponent(v)}');
}
});
final uri = pairs.isEmpty ? base : base.replace(query: pairs.join('&'));
final resp = await _t.send(method, uri, extra: buildHeaders(o));
if (resp.statusCode < 200 || resp.statusCode > 299) {
throw Transport.errorFrom(resp); // 206 is success
}
final text = utf8.decode(resp.bodyBytes);
return Response(
success: true,
data: text.trim().isEmpty ? null : jsonDecode(text),
metadata:
list ? _metadata(_header(resp.headers, 'content-range'), o) : null,
);
}
/// Single-record endpoint (SqlQuery). `data` is the row object.
Future<Response> query(String path,
{Map<String, Object?>? params,
FuncSpecOptions? options,
String method = 'GET'}) =>
_call(method.toUpperCase(), path, params, options, false);
/// List endpoint (SqlQueryList). Metadata comes from Content-Range.
Future<Response> queryList(String path,
{Map<String, Object?>? params,
FuncSpecOptions? options,
String method = 'GET'}) =>
_call(method.toUpperCase(), path, params, options, true);
}
@@ -0,0 +1,88 @@
import 'dart:convert';
import 'client.dart';
import 'types.dart';
/// Client for the ResolveSpec JSON body protocol: POST {operation, data, options}.
///
/// A record `id` of type `int` or `String` goes in the URL; a `List<String>` goes in the body.
class ResolveSpecClient {
final Transport _t;
ResolveSpecClient(String baseUrl, [ClientOptions? options])
: _t = Transport(baseUrl, options);
void close() => _t.close();
static String? _urlId(Object? id) =>
id == null || id is List ? null : id.toString();
static List<String>? _bodyId(Object? id) =>
id is List ? id.map((e) => e.toString()).toList() : null;
Uri _url(String schema, String entity, String? id) {
var u =
'${_t.baseUrl}/${Uri.encodeComponent(schema)}/${Uri.encodeComponent(entity)}';
if (id != null && id.isNotEmpty) u += '/${Uri.encodeComponent(id)}';
return Uri.parse(u);
}
Future<Response> _send(
String method, Uri url, Map<String, dynamic>? body) async {
final resp = await _t.send(method, url,
body: body == null ? null : jsonEncode(body));
if (resp.statusCode < 200 || resp.statusCode > 299) {
throw Transport.errorFrom(resp);
}
final decoded = jsonDecode(utf8.decode(resp.bodyBytes));
final r = Response.fromJson(decoded as Map<String, dynamic>);
if (!r.success && r.error != null) {
throw ResolveSpecException(r.error!.message, resp.statusCode, r.error);
}
return r;
}
/// GET /{schema}/{entity}
Future<Response> getMetadata(String schema, String entity) =>
_send('GET', _url(schema, entity, null), null);
Future<Response> read(String schema, String entity,
{Object? id, Options? options}) =>
_send(
'POST',
_url(schema, entity, _urlId(id)),
{
'operation': 'read',
if (_bodyId(id) != null) 'id': _bodyId(id),
if (options != null) 'options': options.toJson()
},
);
Future<Response> create(String schema, String entity, Object data,
{Options? options}) =>
_send(
'POST',
_url(schema, entity, null),
{
'operation': 'create',
'data': data,
if (options != null) 'options': options.toJson()
},
);
Future<Response> update(String schema, String entity, Object data,
{Object? id, Options? options}) =>
_send(
'POST',
_url(schema, entity, _urlId(id)),
{
'operation': 'update',
if (_bodyId(id) != null) 'id': _bodyId(id),
'data': data,
if (options != null) 'options': options.toJson(),
},
);
Future<Response> delete(String schema, String entity, Object id) =>
_send('POST', _url(schema, entity, _urlId(id)), {'operation': 'delete'});
}
+287
View File
@@ -0,0 +1,287 @@
// Types aligned with Go pkg/common/types.go. toJson() emits the wire names.
Map<String, dynamic> _compact(Map<String, dynamic> m) {
m.removeWhere((_, v) => v == null);
return m;
}
class FilterOption {
final String column;
/// eq neq gt gte lt lte like ilike in contains startswith endswith between
/// between_inclusive is_null is_not_null
final String operator;
final Object? value;
/// AND | OR
final String? logicOperator;
const FilterOption(this.column, this.operator,
[this.value, this.logicOperator]);
Map<String, dynamic> toJson() => _compact({
'column': column,
'operator': operator,
'value': value,
'logic_operator': logicOperator,
});
}
class SortOption {
final String column;
/// asc | desc
final String direction;
const SortOption(this.column, [this.direction = 'asc']);
Map<String, dynamic> toJson() => {'column': column, 'direction': direction};
}
class Parameter {
final String name;
final String value;
final int? sequence;
const Parameter(this.name, this.value, [this.sequence]);
Map<String, dynamic> toJson() =>
_compact({'name': name, 'value': value, 'sequence': sequence});
}
class CustomOperator {
final String name;
final String sql;
const CustomOperator(this.name, this.sql);
Map<String, dynamic> toJson() => {'name': name, 'sql': sql};
}
class ComputedColumn {
final String name;
final String expression;
const ComputedColumn(this.name, this.expression);
Map<String, dynamic> toJson() => {'name': name, 'expression': expression};
}
class PreloadOption {
final String? relation;
final String? tableName;
final List<String>? columns;
final List<String>? omitColumns;
final List<SortOption>? sort;
final List<FilterOption>? filters;
final String? where;
final int? limit;
final int? offset;
final bool? updateable;
final Map<String, String>? computedQl;
final bool? recursive;
final String? primaryKey;
final String? relatedKey;
final String? foreignKey;
final String? recursiveChildKey;
final List<String>? sqlJoins;
final List<String>? joinAliases;
const PreloadOption({
this.relation,
this.tableName,
this.columns,
this.omitColumns,
this.sort,
this.filters,
this.where,
this.limit,
this.offset,
this.updateable,
this.computedQl,
this.recursive,
this.primaryKey,
this.relatedKey,
this.foreignKey,
this.recursiveChildKey,
this.sqlJoins,
this.joinAliases,
});
Map<String, dynamic> toJson() => _compact({
'relation': relation,
'table_name': tableName,
'columns': columns,
'omit_columns': omitColumns,
'sort': sort?.map((e) => e.toJson()).toList(),
'filters': filters?.map((e) => e.toJson()).toList(),
'where': where,
'limit': limit,
'offset': offset,
'updateable': updateable,
'computed_ql': computedQl,
'recursive': recursive,
'primary_key': primaryKey,
'related_key': relatedKey,
'foreign_key': foreignKey,
'recursive_child_key': recursiveChildKey,
'sql_joins': sqlJoins,
'join_aliases': joinAliases,
});
}
class VectorSearchOption {
final String column;
final List<double> vector;
/// l2 (default) | cosine | ip
final String? metric;
/// Distance column alias, default _distance.
final String? as;
final String? direction;
const VectorSearchOption(this.column, this.vector,
{this.metric, this.as, this.direction});
Map<String, dynamic> toJson() => _compact({
'column': column,
'vector': vector,
'metric': metric,
'as': as,
'direction': direction
});
}
/// ResolveSpec request options object.
class Options {
final List<PreloadOption>? preload;
final List<String>? columns;
final List<String>? omitColumns;
final List<FilterOption>? filters;
final List<SortOption>? sort;
final int? limit;
final int? offset;
final List<CustomOperator>? customOperators;
final List<ComputedColumn>? computedColumns;
final List<Parameter>? parameters;
final String? cursorForward;
final String? cursorBackward;
final String? fetchRowNumber;
final VectorSearchOption? vectorSearch;
const Options({
this.preload,
this.columns,
this.omitColumns,
this.filters,
this.sort,
this.limit,
this.offset,
this.customOperators,
this.computedColumns,
this.parameters,
this.cursorForward,
this.cursorBackward,
this.fetchRowNumber,
this.vectorSearch,
});
Map<String, dynamic> toJson() => _compact({
'preload': preload?.map((e) => e.toJson()).toList(),
'columns': columns,
'omit_columns': omitColumns,
'filters': filters?.map((e) => e.toJson()).toList(),
'sort': sort?.map((e) => e.toJson()).toList(),
'limit': limit,
'offset': offset,
'customOperators': customOperators?.map((e) => e.toJson()).toList(),
'computedColumns': computedColumns?.map((e) => e.toJson()).toList(),
'parameters': parameters?.map((e) => e.toJson()).toList(),
'cursor_forward': cursorForward,
'cursor_backward': cursorBackward,
'fetch_row_number': fetchRowNumber,
'vector_search': vectorSearch?.toJson(),
});
}
class Metadata {
final int total;
final int count;
final int filtered;
final int limit;
final int offset;
const Metadata(
{this.total = 0,
this.count = 0,
this.filtered = 0,
this.limit = 0,
this.offset = 0});
factory Metadata.fromJson(Map<String, dynamic> j) => Metadata(
total: (j['total'] as num?)?.toInt() ?? 0,
count: (j['count'] as num?)?.toInt() ?? 0,
filtered: (j['filtered'] as num?)?.toInt() ?? 0,
limit: (j['limit'] as num?)?.toInt() ?? 0,
offset: (j['offset'] as num?)?.toInt() ?? 0,
);
@override
bool operator ==(Object other) =>
other is Metadata &&
other.total == total &&
other.count == count &&
other.filtered == filtered &&
other.limit == limit &&
other.offset == offset;
@override
int get hashCode => Object.hash(total, count, filtered, limit, offset);
@override
String toString() =>
'Metadata(total: $total, count: $count, filtered: $filtered, limit: $limit, offset: $offset)';
}
class ApiError {
final String code;
final String message;
final Object? details;
/// Server-side reason (funcspec / restheadspec).
final String? detail;
final String? sql;
const ApiError(
{this.code = '', this.message = '', this.details, this.detail, this.sql});
factory ApiError.fromJson(Map<String, dynamic> j) => ApiError(
code: (j['code'] as String?) ?? '',
message: (j['message'] as String?) ?? '',
details: j['details'],
detail: j['detail'] as String?,
sql: j['sql'] as String?,
);
}
/// ResolveSpec envelope. [data] is the decoded JSON value (Map, List or scalar).
class Response {
final bool success;
final Object? data;
final Metadata? metadata;
final ApiError? error;
const Response({required this.success, this.data, this.metadata, this.error});
factory Response.fromJson(Map<String, dynamic> j) => Response(
success: j['success'] == true,
data: j['data'],
metadata: j['metadata'] is Map<String, dynamic>
? Metadata.fromJson(j['metadata'] as Map<String, dynamic>)
: null,
error: j['error'] is Map<String, dynamic>
? ApiError.fromJson(j['error'] as Map<String, dynamic>)
: null,
);
}
+14
View File
@@ -0,0 +1,14 @@
name: resolvespec
description: Client for ResolveSpec (JSON body) and FunctionSpec endpoints.
version: 0.1.0
publish_to: none
environment:
sdk: ">=3.3.0 <4.0.0"
dependencies:
http: ^1.2.0
dev_dependencies:
lints: ^4.0.0
test: ^1.25.0
@@ -0,0 +1,216 @@
import 'dart:convert';
import 'package:http/http.dart' as http;
import 'package:http/testing.dart';
import 'package:resolvespec/resolvespec.dart';
import 'package:test/test.dart';
(http.Client, List<http.Request>) stub(int status, Object body,
{Map<String, String> headers = const {}}) {
final seen = <http.Request>[];
final client = MockClient((req) async {
seen.add(req);
final text = body is String ? body : jsonEncode(body);
return http.Response(text, status,
headers: {'content-type': 'application/json', ...headers});
});
return (client, seen);
}
void main() {
group('resolvespec', () {
test('read posts body with headers', () async {
final (c, seen) = stub(200, {
'success': true,
'data': [
{'id': 1}
]
});
final client = ResolveSpecClient(
'http://localhost:3000/',
ClientOptions(token: 'tok', headers: {'X-Tenant': 'a'}, httpClient: c),
);
final r = await client.read('public', 'users',
options:
const Options(limit: 5, filters: [FilterOption('a', 'eq', 1)]));
final req = seen.single;
expect(req.method, 'POST');
expect(req.url.path, '/public/users');
expect(req.headers['authorization'], 'Bearer tok');
expect(req.headers['x-tenant'], 'a');
final body = jsonDecode(req.body) as Map<String, dynamic>;
expect(body['operation'], 'read');
expect(body['options']['limit'], 5);
expect(body.containsKey('id'), isFalse);
expect((r.data as List).length, 1);
});
test('id placement', () async {
final (c, seen) = stub(200, {'success': true, 'data': {}});
final client =
ResolveSpecClient('http://x', ClientOptions(httpClient: c));
await client.read('s', 'e', id: 7);
expect(seen.last.url.path, '/s/e/7');
await client.update('s', 'e', {'a': 1}, id: ['1', '2']);
expect(seen.last.url.path, '/s/e');
final b = jsonDecode(seen.last.body) as Map<String, dynamic>;
expect(b['id'], ['1', '2']);
expect(b['operation'], 'update');
await client.delete('s', 'e', 'a/b');
expect(seen.last.url.toString(), 'http://x/s/e/a%2Fb');
expect(jsonDecode(seen.last.body)['operation'], 'delete');
});
test('errors', () async {
final client = ResolveSpecClient(
'http://x',
ClientOptions(
httpClient: stub(400, {
'success': false,
'error': {'code': 'x', 'message': 'bad', 'detail': 'why'}
}).$1),
);
await expectLater(
client.read('s', 'e'),
throwsA(isA<ResolveSpecException>()
.having((e) => e.statusCode, 'status', 400)
.having((e) => e.error.code, 'code', 'x')
.having((e) => e.message, 'message', 'bad')
.having((e) => e.error.detail, 'detail', 'why')),
);
final plain = ResolveSpecClient(
'http://x', ClientOptions(httpClient: stub(502, 'bad gateway').$1));
await expectLater(
plain.read('s', 'e'),
throwsA(isA<ResolveSpecException>()
.having((e) => e.message, 'message', 'bad gateway')),
);
final soft = ResolveSpecClient(
'http://x',
ClientOptions(
httpClient: stub(200, {
'success': false,
'error': {'code': 'c', 'message': 'nope'}
}).$1),
);
await expectLater(
soft.read('s', 'e'),
throwsA(isA<ResolveSpecException>()
.having((e) => e.message, 'message', 'nope')));
});
});
group('funcspec', () {
test('header filters', () {
final h = buildHeaders(const FuncSpecOptions(filters: [
FilterOption('status', 'eq', 'active'),
FilterOption('age', 'gte', 18),
FilterOption('name', 'contains', 'x', 'OR'),
FilterOption('deleted', 'is_null'),
FilterOption('id', 'in', [1, 2]),
FilterOption('p', 'between_inclusive', [1, 5]),
]));
expect(h, {
'X-FieldFilter-status': 'active',
'X-SearchOp-greaterthanorequal-age': '18',
'X-SearchOr-contains-name': 'x',
'X-SearchOp-empty-deleted': '',
'X-SearchOp-in-id': '1,2',
'X-SearchOp-betweeninclusive-p': '1,5',
});
});
test('misc headers and encoding', () {
var h = buildHeaders(const FuncSpecOptions(
searchFilters: {'name': 'bob'},
customSqlWhere: 'a = 1',
customSqlOr: 'b = 2',
sort: [SortOption('name', 'asc'), SortOption('created_at', 'DESC')],
limit: 5,
offset: 10,
distinct: true,
skipCount: true,
skipCache: false,
responseFormat: 'syncfusion',
));
expect(h['X-Sort'], 'name ASC,created_at DESC');
expect(h['X-SearchFilter-name'], 'bob');
expect(h['X-Custom-SQL-W'], 'a = 1');
expect(h['X-SkipCache'], 'false');
expect(h['X-Syncfusion'], 'true');
h = buildHeaders(const FuncSpecOptions(filters: [
FilterOption('n', 'eq', 'héllo'),
FilterOption('m', 'eq', ' pad'),
]));
expect(h['X-FieldFilter-n'], startsWith('ZIP_'));
expect(decodeHeaderValue(h['X-FieldFilter-n']!), 'héllo');
expect(decodeHeaderValue(h['X-FieldFilter-m']!), ' pad');
});
test('query building', () {
expect(
buildQuery({
'a': true,
'b': ['x', 'y'],
'c': null,
'd': 3
}),
{
'a': ['true'],
'b': ['x', 'y'],
'd': ['3'],
});
});
test('queryList metadata', () async {
final (c, seen) = stub(206, [
{'id': 1},
{'id': 2}
], headers: {
'Content-Range': 'items 10-12/50'
});
final client = FuncSpecClient(
'http://x', ClientOptions(token: 'tok', httpClient: c));
final r = await client.queryList('/api/users',
params: {'org': 1}, options: const FuncSpecOptions(limit: 2));
expect(seen.single.method, 'GET');
expect(seen.single.url.path, '/api/users');
expect(seen.single.url.query, 'org=1');
expect(seen.single.headers['x-limit'], '2');
expect(
r.metadata,
const Metadata(
total: 50, count: 2, filtered: 50, limit: 2, offset: 10));
expect((r.data as List).length, 2);
});
test('query single and error', () async {
final ok = FuncSpecClient(
'http://x', ClientOptions(httpClient: stub(200, {'id': 1}).$1));
final r = await ok.query('api/u');
expect(r.metadata, isNull);
expect((r.data as Map)['id'], 1);
final bad = FuncSpecClient(
'http://x',
ClientOptions(
httpClient: stub(400, {
'success': false,
'error': {
'code': 'hook_error',
'message': 'Hook execution failed',
'detail': 'authentication required'
}
}).$1),
);
await expectLater(
bad.query('api/u'),
throwsA(isA<ResolveSpecException>()
.having((e) => e.error.code, 'code', 'hook_error')
.having(
(e) => e.error.detail, 'detail', 'authentication required')),
);
});
});
}
+40
View File
@@ -0,0 +1,40 @@
# resolvespec-go
Go client for ResolveSpec (JSON body) and FunctionSpec. Module: `github.com/bitechdev/ResolveSpec/clients/resolvespec-go`. Stdlib only.
## Clients
| Type | Constructor | Methods |
|---|---|---|
| `Client` | `NewClient(baseURL, opts...)` | `GetMetadata` `Read` `Create` `Update` `Delete` |
| `FuncSpecClient` | `NewFuncSpecClient(baseURL, opts...)` | `Query` `QueryList` `Do` |
Client options: `WithToken`, `WithHeader`, `WithHTTPClient`. Precedence: Content-Type < custom headers < bearer token.
## ResolveSpec
- All methods take `ctx`; `Read`/`Update`/`Delete` take `RecordID` (`nil`, int/string → URL, `[]string` → body).
- `Options` fields use pointers for optional ints/bools (`Int(n)`, `Bool(b)`).
- Result: `*Response{Success, Data (raw JSON), Metadata}`; `resp.Decode(&v)`.
## FunctionSpec
- Routes are server-defined: pass the `path`.
- `Params` → query string (slice → repeated keys, bool → `true`/`false`).
- `FuncSpecOptions` → `X-*` headers: `Filters`, `SearchFilters`, `CustomSQLWhere`, `CustomSQLOr`, `Sort`, `Limit`, `Offset`, `Distinct`, `SkipCount`, `SkipCache`, `ResponseFormat`.
- `QueryList` fills `Metadata` from `Content-Range` (`items a-b/total`); 206 is success.
## Server quirks
- `Sort` is raw SQL in ORDER BY (client sends `col ASC|DESC`).
- One search operator per column.
- Values starting `ZIP_` / `__` are base64-decoded by the server.
- Non-ASCII, control chars and edge spaces are auto-encoded (`ZIP_`).
## Errors
`*Error{StatusCode, APIError{Code, Message, Detail, SQL}}`.
## Test
`go test ./...`
+121
View File
@@ -0,0 +1,121 @@
package resolvespec
import (
"bytes"
"context"
"encoding/json"
"fmt"
"io"
"net/http"
"net/url"
"strings"
"time"
)
// APIError is the server error object.
type APIError struct {
Code string `json:"code"`
Message string `json:"message"`
Details any `json:"details,omitempty"`
Detail string `json:"detail,omitempty"` // server-side reason (funcspec / restheadspec)
SQL string `json:"sql,omitempty"`
}
// Error is returned on a non-2xx response or an unsuccessful result.
type Error struct {
StatusCode int
APIError
}
func (e *Error) Error() string {
if e.Message != "" {
return e.Message
}
return fmt.Sprintf("http %d", e.StatusCode)
}
type config struct {
baseURL string
token string
headers http.Header
http *http.Client
}
// Option configures a client.
type Option func(*config)
func WithToken(token string) Option { return func(c *config) { c.token = token } }
func WithHTTPClient(h *http.Client) Option { return func(c *config) { c.http = h } }
func WithHeader(name, value string) Option {
return func(c *config) { c.headers.Set(name, value) }
}
func newConfig(baseURL string, opts []Option) config {
c := config{baseURL: strings.TrimRight(baseURL, "/"), headers: http.Header{}, http: &http.Client{Timeout: 30 * time.Second}}
for _, o := range opts {
o(&c)
}
return c
}
// headers: Content-Type < custom headers < bearer token.
func (c *config) newRequest(ctx context.Context, method, u string, body []byte) (*http.Request, error) {
var r io.Reader
if body != nil {
r = bytes.NewReader(body)
}
req, err := http.NewRequestWithContext(ctx, method, u, r)
if err != nil {
return nil, err
}
req.Header.Set("Content-Type", "application/json")
for k, vs := range c.headers {
req.Header[k] = append([]string(nil), vs...)
}
if c.token != "" {
req.Header.Set("Authorization", "Bearer "+c.token)
}
return req, nil
}
func (c *config) do(req *http.Request) (*http.Response, []byte, error) {
resp, err := c.http.Do(req)
if err != nil {
return nil, nil, err
}
defer resp.Body.Close()
b, err := io.ReadAll(resp.Body)
return resp, b, err
}
func errorFrom(status int, body []byte) *Error {
e := &Error{StatusCode: status}
var env struct {
Error *APIError `json:"error"`
}
if json.Unmarshal(body, &env) == nil && env.Error != nil {
e.APIError = *env.Error
}
if e.Message == "" {
text := ""
if !json.Valid(body) {
text = strings.TrimSpace(string(body))
if len(text) > 200 {
text = text[:200]
}
}
if text == "" {
text = fmt.Sprintf("%s (%d)", http.StatusText(status), status)
}
e.Message = text
}
return e
}
func buildURL(base, schema, entity string, id string) string {
u := base + "/" + url.PathEscape(schema) + "/" + url.PathEscape(entity)
if id != "" {
u += "/" + url.PathEscape(id)
}
return u
}
+274
View File
@@ -0,0 +1,274 @@
package resolvespec
import (
"context"
"encoding/base64"
"encoding/json"
"fmt"
"net/http"
"net/url"
"regexp"
"strconv"
"strings"
"unicode"
)
// FuncSpecOptions are sent to funcspec endpoints as X-* headers.
//
// Server behaviour (pkg/funcspec): Sort is inserted raw into ORDER BY (so it is sent as SQL
// terms); only one search operator per column is kept; values starting with "ZIP_" or "__"
// are base64-decoded by the server, so such plaintext values cannot be sent faithfully.
type FuncSpecOptions struct {
Filters []FilterOption // eq+AND -> X-FieldFilter; others X-SearchOp / X-SearchOr
SearchFilters map[string]string // X-SearchFilter-{col}: text ILIKE
CustomSQLWhere string // X-Custom-SQL-W
CustomSQLOr string // X-Custom-SQL-Or
Sort []SortOption
Limit *int
Offset *int
Distinct *bool
SkipCount *bool
SkipCache *bool
ResponseFormat string // simple | detail | syncfusion
}
// Params are query-string values. Slice values are sent as repeated keys (server: IN filter).
type Params map[string]any
// FuncSpecClient calls user-defined SQL endpoints. Routes are defined by the server app.
type FuncSpecClient struct{ cfg config }
func NewFuncSpecClient(baseURL string, opts ...Option) *FuncSpecClient {
return &FuncSpecClient{cfg: newConfig(baseURL, opts)}
}
var operatorMap = map[string]string{
"eq": "equals", "neq": "notequals", "gt": "greaterthan", "gte": "greaterthanorequal",
"lt": "lessthan", "lte": "lessthanorequal", "like": "contains", "ilike": "contains",
"contains": "contains", "startswith": "beginswith", "endswith": "endswith", "in": "in",
"between": "between", "between_inclusive": "betweeninclusive",
"is_null": "empty", "is_not_null": "notempty",
}
func scalar(v any) string {
switch x := v.(type) {
case nil:
return ""
case bool:
return strconv.FormatBool(x)
case string:
return x
case fmt.Stringer:
return x.String()
}
return fmt.Sprint(v)
}
func filterValue(v any) string {
switch x := v.(type) {
case nil:
return ""
case []string:
return strings.Join(x, ",")
case []int:
parts := make([]string, len(x))
for i, n := range x {
parts[i] = strconv.Itoa(n)
}
return strings.Join(parts, ",")
case []any:
parts := make([]string, len(x))
for i, n := range x {
parts[i] = scalar(n)
}
return strings.Join(parts, ",")
}
return scalar(v)
}
// EncodeHeaderValue base64-encodes (UTF-8) with the ZIP_ prefix.
func EncodeHeaderValue(v string) string { return "ZIP_" + base64.StdEncoding.EncodeToString([]byte(v)) }
// DecodeHeaderValue decodes a value that may carry a ZIP_ or __ prefix (nested allowed).
func DecodeHeaderValue(v string) string {
for _, p := range []string{"ZIP_", "__"} {
if strings.HasPrefix(v, p) {
b64 := strings.NewReplacer("\n", "", "\r", "", " ", "").Replace(v[len(p):])
for len(b64)%4 != 0 {
b64 += "="
}
raw, err := base64.StdEncoding.DecodeString(b64)
if err != nil {
return v
}
return DecodeHeaderValue(string(raw))
}
}
return v
}
// safe encodes values that are unsafe as raw header/query text (non-ASCII, control chars, edge spaces).
func safe(v string) string {
if v != strings.TrimSpace(v) {
return EncodeHeaderValue(v)
}
for _, r := range v {
if r > unicode.MaxASCII || !unicode.IsPrint(r) {
return EncodeHeaderValue(v)
}
}
return v
}
// BuildHeaders builds the X-* headers understood by funcspec.ParseParameters.
func BuildHeaders(o *FuncSpecOptions) map[string]string {
h := map[string]string{}
if o == nil {
return h
}
for _, f := range o.Filters {
logic := f.LogicOperator
if logic == "" {
logic = "AND"
}
v := safe(filterValue(f.Value))
if f.Operator == "eq" && logic == "AND" {
h["X-FieldFilter-"+f.Column] = v
continue
}
op := operatorMap[f.Operator]
if op == "" {
op = f.Operator
}
kind := "X-SearchOp"
if logic == "OR" {
kind = "X-SearchOr"
}
h[kind+"-"+op+"-"+f.Column] = v
}
for col, text := range o.SearchFilters {
h["X-SearchFilter-"+col] = safe(text)
}
if o.CustomSQLWhere != "" {
h["X-Custom-SQL-W"] = safe(o.CustomSQLWhere)
}
if o.CustomSQLOr != "" {
h["X-Custom-SQL-Or"] = safe(o.CustomSQLOr)
}
if len(o.Sort) > 0 {
terms := make([]string, len(o.Sort))
for i, s := range o.Sort {
dir := "ASC"
if strings.EqualFold(s.Direction, "desc") {
dir = "DESC"
}
terms[i] = s.Column + " " + dir // funcspec puts this verbatim into ORDER BY
}
h["X-Sort"] = safe(strings.Join(terms, ","))
}
if o.Limit != nil {
h["X-Limit"] = strconv.Itoa(*o.Limit)
}
if o.Offset != nil {
h["X-Offset"] = strconv.Itoa(*o.Offset)
}
for name, v := range map[string]*bool{"X-Distinct": o.Distinct, "X-SkipCount": o.SkipCount, "X-SkipCache": o.SkipCache} {
if v != nil {
h[name] = strconv.FormatBool(*v)
}
}
switch o.ResponseFormat {
case "simple":
h["X-SimpleApi"] = "true"
case "detail":
h["X-DetailApi"] = "true"
case "syncfusion":
h["X-Syncfusion"] = "true"
}
return h
}
// BuildQuery builds query-string values: bools -> true/false, slices -> repeated keys, nil skipped.
func BuildQuery(p Params) url.Values {
q := url.Values{}
for k, v := range p {
switch x := v.(type) {
case nil:
case []string:
for _, e := range x {
q.Add(k, safe(e))
}
case []int:
for _, e := range x {
q.Add(k, strconv.Itoa(e))
}
case []any:
for _, e := range x {
q.Add(k, safe(scalar(e)))
}
default:
q.Add(k, safe(scalar(v)))
}
}
return q
}
var contentRange = regexp.MustCompile(`(\d+)-(\d+)/(\d+)`)
func metadata(h http.Header, o *FuncSpecOptions) *Metadata {
m := &Metadata{}
if g := contentRange.FindStringSubmatch(h.Get("Content-Range")); g != nil {
start, _ := strconv.ParseInt(g[1], 10, 64)
end, _ := strconv.ParseInt(g[2], 10, 64)
total, _ := strconv.ParseInt(g[3], 10, 64)
m.Total, m.Count, m.Filtered, m.Offset = total, end-start, total, int(start)
}
if o != nil && o.Limit != nil {
m.Limit = *o.Limit
}
return m
}
func (c *FuncSpecClient) call(ctx context.Context, method, path string, p Params, o *FuncSpecOptions, withMeta bool) (*Response, error) {
u := c.cfg.baseURL + "/" + strings.TrimLeft(path, "/")
if q := BuildQuery(p); len(q) > 0 {
u += "?" + q.Encode()
}
req, err := c.cfg.newRequest(ctx, strings.ToUpper(method), u, nil)
if err != nil {
return nil, err
}
for k, v := range BuildHeaders(o) {
req.Header.Set(k, v)
}
resp, b, err := c.cfg.do(req)
if err != nil {
return nil, err
}
if resp.StatusCode < 200 || resp.StatusCode > 299 { // 206 is success
return nil, errorFrom(resp.StatusCode, b)
}
out := &Response{Success: true, Data: json.RawMessage(b)}
if len(b) == 0 {
out.Data = json.RawMessage("null")
}
if withMeta {
out.Metadata = metadata(resp.Header, o)
}
return out, nil
}
// Query calls a single-record endpoint (SqlQuery). Data is the row object.
func (c *FuncSpecClient) Query(ctx context.Context, path string, p Params, o *FuncSpecOptions) (*Response, error) {
return c.call(ctx, http.MethodGet, path, p, o, false)
}
// QueryList calls a list endpoint (SqlQueryList). Metadata comes from Content-Range.
func (c *FuncSpecClient) QueryList(ctx context.Context, path string, p Params, o *FuncSpecOptions) (*Response, error) {
return c.call(ctx, http.MethodGet, path, p, o, true)
}
// Do is like Query/QueryList with an explicit HTTP method (routes are app-defined).
func (c *FuncSpecClient) Do(ctx context.Context, method, path string, p Params, o *FuncSpecOptions, list bool) (*Response, error) {
return c.call(ctx, method, path, p, o, list)
}
+98
View File
@@ -0,0 +1,98 @@
package resolvespec
import (
"context"
"reflect"
"testing"
)
func TestBuildHeadersFilters(t *testing.T) {
got := BuildHeaders(&FuncSpecOptions{Filters: []FilterOption{
{Column: "status", Operator: "eq", Value: "active"},
{Column: "age", Operator: "gte", Value: 18},
{Column: "name", Operator: "contains", Value: "x", LogicOperator: "OR"},
{Column: "deleted", Operator: "is_null"},
{Column: "id", Operator: "in", Value: []int{1, 2}},
{Column: "p", Operator: "between_inclusive", Value: []any{1, 5}},
}})
want := map[string]string{
"X-FieldFilter-status": "active",
"X-SearchOp-greaterthanorequal-age": "18",
"X-SearchOr-contains-name": "x",
"X-SearchOp-empty-deleted": "",
"X-SearchOp-in-id": "1,2",
"X-SearchOp-betweeninclusive-p": "1,5",
}
if !reflect.DeepEqual(got, want) {
t.Fatalf("%v", got)
}
}
func TestBuildHeadersMisc(t *testing.T) {
got := BuildHeaders(&FuncSpecOptions{
SearchFilters: map[string]string{"name": "bob"}, CustomSQLWhere: "a = 1", CustomSQLOr: "b = 2",
Sort: []SortOption{{"name", "asc"}, {"created_at", "DESC"}},
Limit: Int(5), Offset: Int(10), Distinct: Bool(true), SkipCount: Bool(true), SkipCache: Bool(false),
ResponseFormat: "syncfusion",
})
want := map[string]string{
"X-SearchFilter-name": "bob", "X-Custom-SQL-W": "a = 1", "X-Custom-SQL-Or": "b = 2",
"X-Sort": "name ASC,created_at DESC", "X-Limit": "5", "X-Offset": "10", "X-Distinct": "true",
"X-SkipCount": "true", "X-SkipCache": "false", "X-Syncfusion": "true",
}
if !reflect.DeepEqual(got, want) {
t.Fatalf("%v", got)
}
}
func TestEncodeUnsafe(t *testing.T) {
h := BuildHeaders(&FuncSpecOptions{Filters: []FilterOption{{Column: "n", Operator: "eq", Value: "héllo"}, {Column: "m", Operator: "eq", Value: " pad"}}})
for _, k := range []string{"X-FieldFilter-n", "X-FieldFilter-m"} {
if len(h[k]) < 4 || h[k][:4] != "ZIP_" {
t.Fatalf("%s=%q", k, h[k])
}
}
if DecodeHeaderValue(h["X-FieldFilter-n"]) != "héllo" || DecodeHeaderValue(h["X-FieldFilter-m"]) != " pad" {
t.Fatal("roundtrip")
}
}
func TestBuildQuery(t *testing.T) {
q := BuildQuery(Params{"a": true, "b": []string{"x", "y"}, "c": nil, "d": 3})
if q.Get("a") != "true" || !reflect.DeepEqual(q["b"], []string{"x", "y"}) || q.Has("c") || q.Get("d") != "3" {
t.Fatalf("%v", q)
}
}
func TestQueryListMetadata(t *testing.T) {
srv, s := server(t, 206, `[{"id":1},{"id":2}]`, map[string]string{"Content-Range": "items 10-12/50"})
c := NewFuncSpecClient(srv.URL, WithToken("tok"))
resp, err := c.QueryList(context.Background(), "/api/users", Params{"org": 1}, &FuncSpecOptions{Limit: Int(2)})
if err != nil {
t.Fatal(err)
}
if s.method != "GET" || s.path != "/api/users?org=1" || s.header.Get("X-Limit") != "2" {
t.Fatalf("%s %v", s.path, s.header)
}
m := resp.Metadata
if m.Total != 50 || m.Count != 2 || m.Offset != 10 || m.Limit != 2 || m.Filtered != 50 {
t.Fatalf("%+v", m)
}
var rows []map[string]any
if err := resp.Decode(&rows); err != nil || len(rows) != 2 {
t.Fatal(err)
}
}
func TestQuerySingleNoMetadataAndError(t *testing.T) {
srv, _ := server(t, 200, `{"id":1}`, nil)
resp, err := NewFuncSpecClient(srv.URL).Query(context.Background(), "api/u", nil, nil)
if err != nil || resp.Metadata != nil {
t.Fatalf("%v %v", resp, err)
}
srv2, _ := server(t, 400, `{"success":false,"error":{"code":"hook_error","message":"Hook execution failed","detail":"authentication required"}}`, nil)
_, err = NewFuncSpecClient(srv2.URL).Query(context.Background(), "api/u", nil, nil)
if e := err.(*Error); e.Code != "hook_error" || e.Detail != "authentication required" {
t.Fatalf("%#v", e)
}
}
+3
View File
@@ -0,0 +1,3 @@
module github.com/bitechdev/ResolveSpec/clients/resolvespec-go
go 1.22
+94
View File
@@ -0,0 +1,94 @@
package resolvespec
import (
"context"
"encoding/json"
"fmt"
"net/http"
)
// Client speaks the ResolveSpec JSON body protocol: POST {operation, data, options}.
type Client struct{ cfg config }
func NewClient(baseURL string, opts ...Option) *Client {
return &Client{cfg: newConfig(baseURL, opts)}
}
// RecordID is a single id (int or string, sent in the URL) or a []string (sent in the body).
type RecordID any
func urlID(id RecordID) string {
switch v := id.(type) {
case nil:
return ""
case []string:
return ""
case string:
return v
default:
return fmt.Sprint(v)
}
}
func bodyID(id RecordID) []string {
ids, _ := id.([]string)
return ids
}
type request struct {
Operation string `json:"operation"`
ID []string `json:"id,omitempty"`
Data any `json:"data,omitempty"`
Options *Options `json:"options,omitempty"`
}
func (c *Client) send(ctx context.Context, method, schema, entity, id string, body any) (*Response, error) {
var payload []byte
if body != nil {
var err error
if payload, err = json.Marshal(body); err != nil {
return nil, err
}
}
req, err := c.cfg.newRequest(ctx, method, buildURL(c.cfg.baseURL, schema, entity, id), payload)
if err != nil {
return nil, err
}
resp, b, err := c.cfg.do(req)
if err != nil {
return nil, err
}
if resp.StatusCode < 200 || resp.StatusCode > 299 {
return nil, errorFrom(resp.StatusCode, b)
}
var out Response
if err := json.Unmarshal(b, &out); err != nil {
return nil, err
}
if !out.Success && out.Error != nil {
return nil, &Error{StatusCode: resp.StatusCode, APIError: *out.Error}
}
return &out, nil
}
// GetMetadata returns table metadata (GET /{schema}/{entity}).
func (c *Client) GetMetadata(ctx context.Context, schema, entity string) (*Response, error) {
return c.send(ctx, http.MethodGet, schema, entity, "", nil)
}
// Read reads records; id may be nil, an int/string (URL) or []string (body).
func (c *Client) Read(ctx context.Context, schema, entity string, id RecordID, opts *Options) (*Response, error) {
return c.send(ctx, http.MethodPost, schema, entity, urlID(id), request{Operation: "read", ID: bodyID(id), Options: opts})
}
func (c *Client) Create(ctx context.Context, schema, entity string, data any, opts *Options) (*Response, error) {
return c.send(ctx, http.MethodPost, schema, entity, "", request{Operation: "create", Data: data, Options: opts})
}
func (c *Client) Update(ctx context.Context, schema, entity string, data any, id RecordID, opts *Options) (*Response, error) {
return c.send(ctx, http.MethodPost, schema, entity, urlID(id), request{Operation: "update", ID: bodyID(id), Data: data, Options: opts})
}
func (c *Client) Delete(ctx context.Context, schema, entity string, id RecordID) (*Response, error) {
return c.send(ctx, http.MethodPost, schema, entity, urlID(id), request{Operation: "delete"})
}
@@ -0,0 +1,99 @@
package resolvespec
import (
"context"
"encoding/json"
"io"
"net/http"
"net/http/httptest"
"reflect"
"testing"
)
type seen struct {
method, path string
header http.Header
body map[string]any
}
func server(t *testing.T, status int, body string, hdr map[string]string) (*httptest.Server, *seen) {
t.Helper()
s := &seen{}
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
s.method, s.path, s.header = r.Method, r.URL.EscapedPath()+"?"+r.URL.RawQuery, r.Header
b, _ := io.ReadAll(r.Body)
if len(b) > 0 {
_ = json.Unmarshal(b, &s.body)
}
for k, v := range hdr {
w.Header().Set(k, v)
}
w.WriteHeader(status)
_, _ = w.Write([]byte(body))
}))
t.Cleanup(srv.Close)
return srv, s
}
func TestReadBody(t *testing.T) {
srv, s := server(t, 200, `{"success":true,"data":[{"id":1}]}`, nil)
c := NewClient(srv.URL+"/", WithToken("tok"), WithHeader("X-Tenant", "a"))
resp, err := c.Read(context.Background(), "public", "users", nil, &Options{Limit: Int(5), Filters: []FilterOption{{Column: "a", Operator: "eq", Value: 1}}})
if err != nil {
t.Fatal(err)
}
if s.method != "POST" || s.path != "/public/users?" {
t.Fatalf("got %s %s", s.method, s.path)
}
if s.header.Get("Authorization") != "Bearer tok" || s.header.Get("X-Tenant") != "a" {
t.Fatalf("headers %v", s.header)
}
if s.body["operation"] != "read" || s.body["options"].(map[string]any)["limit"] != float64(5) {
t.Fatalf("body %v", s.body)
}
var rows []map[string]any
if err := resp.Decode(&rows); err != nil || len(rows) != 1 {
t.Fatalf("decode %v %v", rows, err)
}
}
func TestIDPlacement(t *testing.T) {
srv, s := server(t, 200, `{"success":true,"data":{}}`, nil)
c := NewClient(srv.URL)
ctx := context.Background()
_, _ = c.Read(ctx, "s", "e", 7, nil)
if s.path != "/s/e/7?" || s.body["id"] != nil {
t.Fatalf("%s %v", s.path, s.body)
}
_, _ = c.Update(ctx, "s", "e", map[string]any{"a": 1}, []string{"1", "2"}, nil)
if s.path != "/s/e?" || !reflect.DeepEqual(s.body["id"], []any{"1", "2"}) || s.body["operation"] != "update" {
t.Fatalf("%s %v", s.path, s.body)
}
_, _ = c.Delete(ctx, "s", "e", "a/b")
if s.path != "/s/e/a%2Fb?" || s.body["operation"] != "delete" {
t.Fatalf("%s %v", s.path, s.body)
}
_, _ = c.GetMetadata(ctx, "s", "e")
if s.method != "GET" {
t.Fatal(s.method)
}
}
func TestErrors(t *testing.T) {
srv, _ := server(t, 400, `{"success":false,"error":{"code":"x","message":"bad","detail":"why"}}`, nil)
_, err := NewClient(srv.URL).Read(context.Background(), "s", "e", nil, nil)
e, ok := err.(*Error)
if !ok || e.StatusCode != 400 || e.Code != "x" || e.Message != "bad" || e.Detail != "why" {
t.Fatalf("%#v", err)
}
srv2, _ := server(t, 502, "bad gateway", nil)
_, err = NewClient(srv2.URL).Read(context.Background(), "s", "e", nil, nil)
if e := err.(*Error); e.StatusCode != 502 || e.Message != "bad gateway" {
t.Fatalf("%#v", e)
}
srv3, _ := server(t, 200, `{"success":false,"error":{"code":"c","message":"nope"}}`, nil)
_, err = NewClient(srv3.URL).Read(context.Background(), "s", "e", nil, nil)
if e := err.(*Error); e.Message != "nope" {
t.Fatalf("%#v", e)
}
}
+107
View File
@@ -0,0 +1,107 @@
// Package resolvespec is a client for ResolveSpec (JSON body) and FunctionSpec endpoints.
package resolvespec
import "encoding/json"
// FilterOption mirrors common.FilterOption. Operator: eq neq gt gte lt lte like ilike in
// contains startswith endswith between between_inclusive is_null is_not_null.
type FilterOption struct {
Column string `json:"column"`
Operator string `json:"operator"`
Value any `json:"value"`
LogicOperator string `json:"logic_operator,omitempty"` // AND | OR
}
type SortOption struct {
Column string `json:"column"`
Direction string `json:"direction"` // asc | desc
}
type Parameter struct {
Name string `json:"name"`
Value string `json:"value"`
Sequence int `json:"sequence,omitempty"`
}
type CustomOperator struct {
Name string `json:"name"`
SQL string `json:"sql"`
}
type ComputedColumn struct {
Name string `json:"name"`
Expression string `json:"expression"`
}
type PreloadOption struct {
Relation string `json:"relation,omitempty"`
TableName string `json:"table_name,omitempty"`
Columns []string `json:"columns,omitempty"`
OmitColumns []string `json:"omit_columns,omitempty"`
Sort []SortOption `json:"sort,omitempty"`
Filters []FilterOption `json:"filters,omitempty"`
Where string `json:"where,omitempty"`
Limit *int `json:"limit,omitempty"`
Offset *int `json:"offset,omitempty"`
Updateable *bool `json:"updateable,omitempty"`
ComputedQL map[string]string `json:"computed_ql,omitempty"`
Recursive bool `json:"recursive,omitempty"`
PrimaryKey string `json:"primary_key,omitempty"`
RelatedKey string `json:"related_key,omitempty"`
ForeignKey string `json:"foreign_key,omitempty"`
RecursiveChildKey string `json:"recursive_child_key,omitempty"`
SQLJoins []string `json:"sql_joins,omitempty"`
JoinAliases []string `json:"join_aliases,omitempty"`
}
type VectorSearchOption struct {
Column string `json:"column"`
Vector []float64 `json:"vector"`
Metric string `json:"metric,omitempty"` // l2 (default) | cosine | ip
As string `json:"as,omitempty"` // distance alias, default _distance
Direction string `json:"direction,omitempty"`
}
// Options is the ResolveSpec request options object.
type Options struct {
Preload []PreloadOption `json:"preload,omitempty"`
Columns []string `json:"columns,omitempty"`
OmitColumns []string `json:"omit_columns,omitempty"`
Filters []FilterOption `json:"filters,omitempty"`
Sort []SortOption `json:"sort,omitempty"`
Limit *int `json:"limit,omitempty"`
Offset *int `json:"offset,omitempty"`
CustomOperators []CustomOperator `json:"customOperators,omitempty"`
ComputedColumns []ComputedColumn `json:"computedColumns,omitempty"`
Parameters []Parameter `json:"parameters,omitempty"`
CursorForward string `json:"cursor_forward,omitempty"`
CursorBackward string `json:"cursor_backward,omitempty"`
FetchRowNumber string `json:"fetch_row_number,omitempty"`
VectorSearch *VectorSearchOption `json:"vector_search,omitempty"`
}
// Metadata of a list response.
type Metadata struct {
Total int64 `json:"total"`
Count int64 `json:"count"`
Filtered int64 `json:"filtered"`
Limit int `json:"limit"`
Offset int `json:"offset"`
}
// Response is the ResolveSpec envelope. Data is left raw for the caller to decode.
type Response struct {
Success bool `json:"success"`
Data json.RawMessage `json:"data"`
Metadata *Metadata `json:"metadata,omitempty"`
Error *APIError `json:"error,omitempty"`
}
// Decode unmarshals Data into v.
func (r *Response) Decode(v any) error { return json.Unmarshal(r.Data, v) }
// Int returns a pointer to n, for optional Options fields.
func Int(n int) *int { return &n }
// Bool returns a pointer to b.
func Bool(b bool) *bool { return &b }
+14
View File
@@ -0,0 +1,14 @@
# @warkypublic/resolvespec-js
## 1.0.2
### Patch Changes
- b587cbd: Forward custom ClientConfig headers on every ResolveSpec and HeaderSpec request. Merge headers case-insensitively and isolate cached clients by URL and effective headers, including authentication and tenant headers.
- 7f8982f: fix: added headers and few fixes
## 1.0.1
### Patch Changes
- Fixed headerpsec
@@ -28,7 +28,7 @@ import { ResolveSpecClient, getResolveSpecClient } from '@warkypublic/resolvespe
// Class instantiation
const client = new ResolveSpecClient({ baseUrl: 'http://localhost:3000', token: 'your-token' });
// Or singleton factory (returns cached instance per baseUrl)
// Or singleton factory (returns cached instance per baseUrl and effective headers)
const client = getResolveSpecClient({ baseUrl: 'http://localhost:3000', token: 'your-token' });
// Read with filters, sort, pagination
@@ -106,6 +106,25 @@ await client.delete('public', 'users', '42');
| `X-Fetch-RowNumber` | `fetch_row_number` | string |
| `X-CQL-SEL-{col}` | `computedColumns` | expression |
| `X-Custom-SQL-W` | `customOperators` | SQL AND-joined |
| `X-Preload-Where` | `preload[].where` | applies to all preloads in `X-Preload`; differing wheres go to `X-Preload-{n}` + `X-Preload-{n}-Where` |
| `X-Expand` | `expand` | `Rel:col1,col2` pipe-separated (LEFT JOIN) |
| `X-Custom-SQL-Join` | `custom_sql_joins` | JOIN clauses, pipe-separated |
| `X-Custom-SQL-Or` | `custom_sql_or` | SQL OR-joined |
| `X-SearchCols` | `search_columns` | comma-separated |
| `X-AdvSQL-{col}` | `advanced_sql` | column -> SQL |
| `X-SpatialFilter-{col}` | `filters` (`st_dwithin`, `st_*`, `bbox`) | JSON `{op,value,logic}` |
| `X-VectorFilter-{col}` | `filters` (`l2_within`, `cosine_within`, `ip_within`) | JSON `{op,value,logic}` |
| `X-Vector-Search-{col}` / `-Vector` / `-As` / `-Dir` | `vector_search` | metric / JSON array / alias / asc\|desc |
| `X-Clean-JSON` | `clean_json` | bool |
| `X-Distinct` | `distinct` | bool |
| `X-SkipCount` / `X-SkipCache` | `skip_count` / `skip_cache` | bool |
| `X-PKRow` | `pk_row` | string |
| `X-SimpleApi` / `X-DetailApi` / `X-Syncfusion` | `response_format` | `simple` \| `detail` \| `syncfusion` |
| `X-Single-Record-As-Object` | `single_record_as_object` | bool (server default true) |
| `X-Transaction-Atomic` | `atomic_transaction` | bool |
| `X-Files` | `xfiles` | JSON, sent as `ZIP_` base64 |
Extended fields live on `HeaderSpecOptions` (extends `Options`); `vector_search` is on `Options`.
### Utility Functions
@@ -211,3 +230,25 @@ pnpm run lint # eslint
## License
MIT
### Custom HTTP headers
Both `ResolveSpecClient` and `HeaderSpecClient` (including their factory functions)
accept `headers` in `ClientConfig` and send them on every HTTP request:
```typescript
const client = new ResolveSpecClient({
baseUrl: 'http://localhost:3000',
token: 'your-token',
headers: { 'X-Tenant': 'acme' },
});
```
Header names are merged case-insensitively. Custom headers override the default
`Content-Type`; a supplied `token` overrides custom `Authorization`, and HeaderSpec
query options override matching custom query headers. Without a token, custom
`Authorization` is preserved. Configuration is copied at construction; create or
retrieve a client with new configuration to change headers. Factory clients are
cached by URL and effective headers, keeping different tenants and tokens separate.
Grid adapters must forward `dataSourceOptions.headers` to this `headers` option.
File diff suppressed because one or more lines are too long
+5
View File
@@ -0,0 +1,5 @@
export * from './common';
export * from './resolvespec';
export * from './websocketspec';
export * from './headerspec';
//# sourceMappingURL=index.d.ts.map
+432
View File
@@ -0,0 +1,432 @@
import { v4 as e } from "uuid";
import { b64DecodeUnicode as t, b64EncodeUnicode as n } from "@warkypublic/artemis-kit/base64";
//#region src/common/http.ts
function r(...e) {
let t = {};
for (let n of e) for (let [e, r] of Object.entries(n)) {
for (let n of Object.keys(t)) n.toLowerCase() === e.toLowerCase() && delete t[n];
Object.defineProperty(t, e, {
value: r,
enumerable: !0,
configurable: !0,
writable: !0
});
}
return t;
}
function i(e) {
return r({ "Content-Type": "application/json" }, e.headers ?? {}, e.token ? { Authorization: `Bearer ${e.token}` } : {});
}
function a(e) {
let t = Object.entries(i(e)).map(([e, t]) => [e.toLowerCase(), t]).sort(([e], [t]) => e.localeCompare(t));
return JSON.stringify([e.baseUrl, t]);
}
//#endregion
//#region src/resolvespec/client.ts
var o = /* @__PURE__ */ new Map();
function s(e) {
let t = a(e), n = o.get(t);
return n || (n = new c(e), o.set(t, n)), n;
}
var c = class {
constructor(e) {
this.config = {
...e,
headers: { ...e.headers }
};
}
buildUrl(e, t, n) {
let r = `${this.config.baseUrl}/${e}/${t}`;
return n && (r += `/${n}`), r;
}
baseHeaders() {
return i(this.config);
}
async fetchWithError(e, t) {
let n = await fetch(e, t), r = await n.json();
if (!n.ok) throw Error(r.error?.message || "An error occurred");
return r;
}
async getMetadata(e, t) {
let n = this.buildUrl(e, t);
return this.fetchWithError(n, {
method: "GET",
headers: this.baseHeaders()
});
}
async read(e, t, n, r) {
let i = typeof n == "number" || typeof n == "string" ? String(n) : void 0, a = this.buildUrl(e, t, i), o = {
operation: "read",
id: Array.isArray(n) ? n : void 0,
options: r
};
return this.fetchWithError(a, {
method: "POST",
headers: this.baseHeaders(),
body: JSON.stringify(o)
});
}
async create(e, t, n, r) {
let i = this.buildUrl(e, t), a = {
operation: "create",
data: n,
options: r
};
return this.fetchWithError(i, {
method: "POST",
headers: this.baseHeaders(),
body: JSON.stringify(a)
});
}
async update(e, t, n, r, i) {
let a = typeof r == "number" || typeof r == "string" ? String(r) : void 0, o = this.buildUrl(e, t, a), s = {
operation: "update",
id: Array.isArray(r) ? r : void 0,
data: n,
options: i
};
return this.fetchWithError(o, {
method: "POST",
headers: this.baseHeaders(),
body: JSON.stringify(s)
});
}
async delete(e, t, n) {
let r = this.buildUrl(e, t, String(n));
return this.fetchWithError(r, {
method: "POST",
headers: this.baseHeaders(),
body: JSON.stringify({ operation: "delete" })
});
}
}, l = /* @__PURE__ */ new Map();
function u(e) {
let t = e.url, n = l.get(t);
return n || (n = new d(e), l.set(t, n)), n;
}
var d = class {
constructor(e) {
this.ws = null, this.messageHandlers = /* @__PURE__ */ new Map(), this.subscriptions = /* @__PURE__ */ new Map(), this.eventListeners = {}, this.state = "disconnected", this.reconnectAttempts = 0, this.reconnectTimer = null, this.heartbeatTimer = null, this.isManualClose = !1, this.config = {
url: e.url,
reconnect: e.reconnect ?? !0,
reconnectInterval: e.reconnectInterval ?? 3e3,
maxReconnectAttempts: e.maxReconnectAttempts ?? 10,
heartbeatInterval: e.heartbeatInterval ?? 3e4,
debug: e.debug ?? !1
};
}
async connect() {
if (this.ws?.readyState === WebSocket.OPEN) {
this.log("Already connected");
return;
}
return this.isManualClose = !1, this.setState("connecting"), new Promise((e, t) => {
try {
this.ws = new WebSocket(this.config.url), this.ws.onopen = () => {
this.log("Connected to WebSocket server"), this.setState("connected"), this.reconnectAttempts = 0, this.startHeartbeat(), this.emit("connect"), e();
}, this.ws.onmessage = (e) => {
this.handleMessage(e.data);
}, this.ws.onerror = (e) => {
this.log("WebSocket error:", e);
let n = /* @__PURE__ */ Error("WebSocket connection error");
this.emit("error", n), t(n);
}, this.ws.onclose = (e) => {
this.log("WebSocket closed:", e.code, e.reason), this.stopHeartbeat(), this.setState("disconnected"), this.emit("disconnect", e), this.config.reconnect && !this.isManualClose && this.reconnectAttempts < this.config.maxReconnectAttempts && (this.reconnectAttempts++, this.log(`Reconnection attempt ${this.reconnectAttempts}/${this.config.maxReconnectAttempts}`), this.setState("reconnecting"), this.reconnectTimer = setTimeout(() => {
this.connect().catch((e) => {
this.log("Reconnection failed:", e);
});
}, this.config.reconnectInterval));
};
} catch (e) {
t(e);
}
});
}
disconnect() {
this.isManualClose = !0, this.reconnectTimer &&= (clearTimeout(this.reconnectTimer), null), this.stopHeartbeat(), this.ws &&= (this.setState("disconnecting"), this.ws.close(), null), this.setState("disconnected"), this.messageHandlers.clear();
}
async request(t, n, r) {
this.ensureConnected();
let i = e(), a = {
id: i,
type: "request",
operation: t,
entity: n,
schema: r?.schema,
record_id: r?.record_id,
data: r?.data,
options: r?.options
};
return new Promise((e, t) => {
this.messageHandlers.set(i, (n) => {
n.success ? e(n.data) : t(Error(n.error?.message || "Request failed"));
}), this.send(a), setTimeout(() => {
this.messageHandlers.has(i) && (this.messageHandlers.delete(i), t(/* @__PURE__ */ Error("Request timeout")));
}, 3e4);
});
}
async read(e, t) {
return this.request("read", e, {
schema: t?.schema,
record_id: t?.record_id,
options: {
filters: t?.filters,
columns: t?.columns,
sort: t?.sort,
preload: t?.preload,
limit: t?.limit,
offset: t?.offset
}
});
}
async create(e, t, n) {
return this.request("create", e, {
schema: n?.schema,
data: t
});
}
async update(e, t, n, r) {
return this.request("update", e, {
schema: r?.schema,
record_id: t,
data: n
});
}
async delete(e, t, n) {
await this.request("delete", e, {
schema: n?.schema,
record_id: t
});
}
async meta(e, t) {
return this.request("meta", e, { schema: t?.schema });
}
async subscribe(t, n, r) {
this.ensureConnected();
let i = e(), a = {
id: i,
type: "subscription",
operation: "subscribe",
entity: t,
schema: r?.schema,
options: { filters: r?.filters }
};
return new Promise((e, o) => {
this.messageHandlers.set(i, (i) => {
if (i.success && i.data?.subscription_id) {
let a = i.data.subscription_id;
this.subscriptions.set(a, {
id: a,
entity: t,
schema: r?.schema,
options: { filters: r?.filters },
callback: n
}), this.log(`Subscribed to ${t} with ID: ${a}`), e(a);
} else o(Error(i.error?.message || "Subscription failed"));
}), this.send(a), setTimeout(() => {
this.messageHandlers.has(i) && (this.messageHandlers.delete(i), o(/* @__PURE__ */ Error("Subscription timeout")));
}, 1e4);
});
}
async unsubscribe(t) {
this.ensureConnected();
let n = e(), r = {
id: n,
type: "subscription",
operation: "unsubscribe",
subscription_id: t
};
return new Promise((e, i) => {
this.messageHandlers.set(n, (n) => {
n.success ? (this.subscriptions.delete(t), this.log(`Unsubscribed from ${t}`), e()) : i(Error(n.error?.message || "Unsubscribe failed"));
}), this.send(r), setTimeout(() => {
this.messageHandlers.has(n) && (this.messageHandlers.delete(n), i(/* @__PURE__ */ Error("Unsubscribe timeout")));
}, 1e4);
});
}
getSubscriptions() {
return Array.from(this.subscriptions.values());
}
getState() {
return this.state;
}
isConnected() {
return this.ws?.readyState === WebSocket.OPEN;
}
on(e, t) {
this.eventListeners[e] = t;
}
off(e) {
delete this.eventListeners[e];
}
handleMessage(e) {
try {
let t = JSON.parse(e);
switch (this.log("Received message:", t), this.emit("message", t), t.type) {
case "response":
this.handleResponse(t);
break;
case "notification":
this.handleNotification(t);
break;
case "pong": break;
default: this.log("Unknown message type:", t.type);
}
} catch (e) {
this.log("Error parsing message:", e);
}
}
handleResponse(e) {
let t = this.messageHandlers.get(e.id);
t && (t(e), this.messageHandlers.delete(e.id));
}
handleNotification(e) {
let t = this.subscriptions.get(e.subscription_id);
t?.callback && t.callback(e);
}
send(e) {
if (!this.ws || this.ws.readyState !== WebSocket.OPEN) throw Error("WebSocket is not connected");
let t = JSON.stringify(e);
this.log("Sending message:", e), this.ws.send(t);
}
startHeartbeat() {
this.heartbeatTimer ||= setInterval(() => {
if (this.isConnected()) {
let t = {
id: e(),
type: "ping"
};
this.send(t);
}
}, this.config.heartbeatInterval);
}
stopHeartbeat() {
this.heartbeatTimer &&= (clearInterval(this.heartbeatTimer), null);
}
setState(e) {
this.state !== e && (this.state = e, this.emit("stateChange", e));
}
ensureConnected() {
if (!this.isConnected()) throw Error("WebSocket is not connected. Call connect() first.");
}
emit(e, ...t) {
let n = this.eventListeners[e];
n && n(...t);
}
log(...e) {
this.config.debug && console.log("[WebSocketClient]", ...e);
}
};
//#endregion
//#region src/headerspec/client.ts
function f(e) {
return "ZIP_" + n(e);
}
function p(e) {
let t = e;
return t.startsWith("ZIP_") ? (t = t.slice(4).replace(/[\n\r ]/g, ""), t = m(t)) : t.startsWith("__") && (t = t.slice(2).replace(/[\n\r ]/g, ""), t = m(t)), (t.startsWith("ZIP_") || t.startsWith("__")) && (t = p(t)), t;
}
function m(e) {
return t(e);
}
function h(e) {
let t = {};
if (e.columns?.length && (t["X-Select-Fields"] = e.columns.join(",")), e.omit_columns?.length && (t["X-Not-Select-Fields"] = e.omit_columns.join(",")), e.filters?.length) for (let n of e.filters) {
let e = n.logic_operator ?? "AND", r = g(n.operator), i = _(n);
n.operator === "eq" && e === "AND" ? t[`X-FieldFilter-${n.column}`] = i : e === "OR" ? t[`X-SearchOr-${r}-${n.column}`] = i : t[`X-SearchOp-${r}-${n.column}`] = i;
}
if (e.sort?.length && (t["X-Sort"] = e.sort.map((e) => e.direction.toUpperCase() === "DESC" ? `-${e.column}` : `+${e.column}`).join(",")), e.limit !== void 0 && (t["X-Limit"] = String(e.limit)), e.offset !== void 0 && (t["X-Offset"] = String(e.offset)), e.cursor_forward && (t["X-Cursor-Forward"] = e.cursor_forward), e.cursor_backward && (t["X-Cursor-Backward"] = e.cursor_backward), e.preload?.length && (t["X-Preload"] = e.preload.map((e) => e.columns?.length ? `${e.relation}:${e.columns.join(",")}` : e.relation).join("|")), e.fetch_row_number && (t["X-Fetch-RowNumber"] = e.fetch_row_number), e.computedColumns?.length) for (let n of e.computedColumns) t[`X-CQL-SEL-${n.name}`] = n.expression;
return e.customOperators?.length && (t["X-Custom-SQL-W"] = e.customOperators.map((e) => e.sql).join(" AND ")), t;
}
function g(e) {
switch (e) {
case "eq": return "equals";
case "neq": return "notequals";
case "gt": return "greaterthan";
case "gte": return "greaterthanorequal";
case "lt": return "lessthan";
case "lte": return "lessthanorequal";
case "like":
case "ilike":
case "contains": return "contains";
case "startswith": return "beginswith";
case "endswith": return "endswith";
case "in": return "in";
case "between": return "between";
case "between_inclusive": return "betweeninclusive";
case "is_null": return "empty";
case "is_not_null": return "notempty";
default: return e;
}
}
function _(e) {
return e.value === null || e.value === void 0 ? "" : Array.isArray(e.value) ? e.value.join(",") : String(e.value);
}
var v = /* @__PURE__ */ new Map();
function y(e) {
let t = a(e), n = v.get(t);
return n || (n = new b(e), v.set(t, n)), n;
}
var b = class {
constructor(e) {
this.config = {
...e,
headers: { ...e.headers }
};
}
buildUrl(e, t, n) {
let r = `${this.config.baseUrl}/${e}/${t}`;
return n && (r += `/${n}`), r;
}
baseHeaders() {
return i(this.config);
}
async fetchWithError(e, t) {
let n = await fetch(e, t), r = await n.json();
if (!n.ok) throw Error(r.error?.message || `${n.statusText} (${n.status})`);
return {
data: r,
success: !0,
error: r.error ? r.error : void 0,
metadata: {
count: n.headers.get("content-range") ? Number(n.headers.get("content-range")?.split("/")[1]) : 0,
total: n.headers.get("content-range") ? Number(n.headers.get("content-range")?.split("/")[1]) : 0,
filtered: n.headers.get("content-range") ? Number(n.headers.get("content-range")?.split("/")[1]) : 0,
offset: n.headers.get("content-range") ? Number(n.headers.get("content-range")?.split("/")[0].split("-")[0]) : 0,
limit: n.headers.get("x-limit") ? Number(n.headers.get("x-limit")) : 0
}
};
}
async read(e, t, n, i) {
let a = this.buildUrl(e, t, n), o = i ? h(i) : {};
return this.fetchWithError(a, {
method: "GET",
headers: r(this.baseHeaders(), o)
});
}
async create(e, t, n, i) {
let a = this.buildUrl(e, t), o = i ? h(i) : {};
return this.fetchWithError(a, {
method: "POST",
headers: r(this.baseHeaders(), o),
body: JSON.stringify(n)
});
}
async update(e, t, n, i, a) {
let o = this.buildUrl(e, t, n), s = a ? h(a) : {};
return this.fetchWithError(o, {
method: "PUT",
headers: r(this.baseHeaders(), s),
body: JSON.stringify(i)
});
}
async delete(e, t, n) {
let r = this.buildUrl(e, t, n);
return this.fetchWithError(r, {
method: "DELETE",
headers: this.baseHeaders()
});
}
};
//#endregion
export { b as HeaderSpecClient, c as ResolveSpecClient, d as WebSocketClient, h as buildHeaders, p as decodeHeaderValue, f as encodeHeaderValue, y as getHeaderSpecClient, s as getResolveSpecClient, u as getWebSocketClient };
@@ -1,6 +1,6 @@
{
"name": "@warkypublic/resolvespec-js",
"version": "1.0.1",
"version": "1.0.2",
"description": "TypeScript client library for ResolveSpec REST, HeaderSpec, and WebSocket APIs",
"type": "module",
"main": "./dist/index.cjs",
@@ -38,20 +38,22 @@
"author": "Hein (Warkanum) Puth",
"license": "MIT",
"dependencies": {
"uuid": "^13.0.0"
"@warkypublic/artemis-kit": "^1.0.10",
"uuid": "^14.0.2"
},
"devDependencies": {
"@changesets/cli": "^2.29.8",
"@changesets/cli": "^3.0.3",
"@eslint/js": "^10.0.1",
"@types/jsdom": "^27.0.0",
"eslint": "^10.0.0",
"globals": "^17.3.0",
"jsdom": "^28.1.0",
"typescript": "^5.9.3",
"typescript-eslint": "^8.55.0",
"vite": "^7.3.1",
"vite-plugin-dts": "^4.5.4",
"vitest": "^4.0.18"
"@types/jsdom": "^30.0.0",
"@types/node": "^26.6.2",
"eslint": "^10.11.0",
"globals": "^17.12.0",
"jsdom": "^30.1.1",
"typescript": "^6.0.3",
"typescript-eslint": "^8.70.1",
"vite": "^8.3.0",
"vite-plugin-dts": "^5.1.1",
"vitest": "^5.0.1"
},
"engines": {
"node": ">=18"
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,5 @@
packages:
- '.'
allowBuilds:
esbuild: true
@@ -0,0 +1,65 @@
import { afterEach, describe, expect, it, vi } from 'vitest';
import { ResolveSpecClient, getResolveSpecClient } from '../resolvespec/client';
import { HeaderSpecClient, getHeaderSpecClient } from '../headerspec/client';
afterEach(() => vi.unstubAllGlobals());
for (const [name, Client, factory] of [
['ResolveSpec', ResolveSpecClient, getResolveSpecClient],
['HeaderSpec', HeaderSpecClient, getHeaderSpecClient],
] as const) {
describe(`${name} custom headers`, () => {
it('sends tenant headers on every operation and resolves collisions case-insensitively', async () => {
const fetchMock = vi.fn().mockResolvedValue({
ok: true, headers: new Headers(), json: async () => ({ success: true, data: [] }),
});
vi.stubGlobal('fetch', fetchMock);
const headers = { 'X-Tenant': 'acme', authorization: 'Basic ignored', 'content-type': 'application/custom+json', 'x-limit': '99' };
const client = new Client({ baseUrl: 'http://localhost:3000', token: 'tok', headers });
await client.read('public', 'users', undefined, { limit: 10 });
await client.create('public', 'users', {});
if (client instanceof ResolveSpecClient) {
await client.update('public', 'users', {}, '1');
await client.getMetadata('public', 'users');
} else {
await client.update('public', 'users', '1', {});
}
await client.delete('public', 'users', '1');
for (const [, init] of fetchMock.mock.calls) {
const sent = new Headers(init.headers);
expect(sent.get('x-tenant')).toBe('acme');
expect(sent.get('authorization')).toBe('Bearer tok');
expect(sent.get('content-type')).toBe('application/custom+json');
}
if (client instanceof HeaderSpecClient) {
expect(new Headers(fetchMock.mock.calls[0][1].headers).get('x-limit')).toBe('10');
}
expect(headers.authorization).toBe('Basic ignored');
expect(headers['x-limit']).toBe('99');
});
it('supports custom authentication without a token', async () => {
const fetchMock = vi.fn().mockResolvedValue({
ok: true, headers: new Headers(), json: async () => ({ success: true, data: [] }),
});
vi.stubGlobal('fetch', fetchMock);
await new Client({ baseUrl: 'http://localhost:3000', headers: { Authorization: 'Basic custom' } }).read('public', 'users');
expect(new Headers(fetchMock.mock.calls[0][1].headers).get('authorization')).toBe('Basic custom');
});
it('isolates cached clients by headers and token, and snapshots configuration', async () => {
const config = { baseUrl: 'http://tenant-cache', token: 'one', headers: { 'X-Tenant': 'acme', 'X-App': 'grid' } };
const first = factory(config);
expect(factory({ ...config, headers: { 'x-app': 'grid', 'x-tenant': 'acme' } })).toBe(first);
expect(factory({ ...config, token: 'two' })).not.toBe(first);
config.headers['X-Tenant'] = 'other';
expect(factory(config)).not.toBe(first);
const fetchMock = vi.fn().mockResolvedValue({
ok: true, headers: new Headers(), json: async () => ({ success: true, data: [] }),
});
vi.stubGlobal('fetch', fetchMock);
await first.read('public', 'users');
expect(new Headers(fetchMock.mock.calls[0][1].headers).get('x-tenant')).toBe('acme');
});
});
}
@@ -2,6 +2,101 @@ import { describe, it, expect, vi, beforeEach } from 'vitest';
import { buildHeaders, encodeHeaderValue, decodeHeaderValue, HeaderSpecClient, getHeaderSpecClient } from '../headerspec/client';
import type { Options, ClientConfig, APIResponse } from '../common/types';
describe('buildHeaders (extended restheadspec options)', () => {
it('should set X-Preload-Where when all preloads share one where', () => {
const h = buildHeaders({
preload: [
{ relation: 'Items', columns: ['id'], where: 'active = true' },
{ relation: 'Tags', where: 'active = true' },
],
});
expect(h['X-Preload']).toBe('Items:id|Tags');
expect(h['X-Preload-Where']).toBe('active = true');
});
it('should use numbered headers for mixed where clauses', () => {
const h = buildHeaders({
preload: [
{ relation: 'Items', where: 'a = 1' },
{ relation: 'Category' },
{ relation: 'Tags', where: 'b = 2' },
],
});
expect(h['X-Preload']).toBe('Category');
expect(h['X-Preload-Where']).toBeUndefined();
expect(h['X-Preload-1']).toBe('Items');
expect(h['X-Preload-1-Where']).toBe('a = 1');
expect(h['X-Preload-2']).toBe('Tags');
expect(h['X-Preload-2-Where']).toBe('b = 2');
});
it('should set expand, joins, or-sql, search cols, advsql', () => {
const h = buildHeaders({
expand: [{ relation: 'Dept', columns: ['id', 'name'] }, { relation: 'Role' }],
custom_sql_joins: ['LEFT JOIN a ON a.id = b.id', 'INNER JOIN c ON c.id = b.cid'],
custom_sql_or: ['x = 1', 'y = 2'],
search_columns: ['name', 'email'],
advanced_sql: { total: 'a + b' },
});
expect(h['X-Expand']).toBe('Dept:id,name|Role');
expect(h['X-Custom-SQL-Join']).toBe('LEFT JOIN a ON a.id = b.id|INNER JOIN c ON c.id = b.cid');
expect(h['X-Custom-SQL-Or']).toBe('x = 1 OR y = 2');
expect(h['X-SearchCols']).toBe('name,email');
expect(h['X-AdvSQL-total']).toBe('a + b');
});
it('should set boolean flags, pk row and response format', () => {
const h = buildHeaders({
clean_json: true,
distinct: true,
skip_count: true,
skip_cache: false,
atomic_transaction: true,
single_record_as_object: false,
pk_row: '42',
response_format: 'detail',
});
expect(h['X-Clean-JSON']).toBe('true');
expect(h['X-Distinct']).toBe('true');
expect(h['X-SkipCount']).toBe('true');
expect(h['X-SkipCache']).toBe('false');
expect(h['X-Transaction-Atomic']).toBe('true');
expect(h['X-Single-Record-As-Object']).toBe('false');
expect(h['X-PKRow']).toBe('42');
expect(h['X-DetailApi']).toBe('true');
});
it('should set spatial and vector filters as JSON', () => {
const h = buildHeaders({
filters: [
{ column: 'geom', operator: 'st_dwithin', value: { geom: 'POINT(0 0)', distance: 5 }, logic_operator: 'OR' },
{ column: 'emb', operator: 'cosine_within', value: { vector: [1, 2], distance: 0.3 } },
],
});
expect(JSON.parse(h['X-SpatialFilter-geom'])).toEqual({
op: 'st_dwithin', value: { geom: 'POINT(0 0)', distance: 5 }, logic: 'or',
});
expect(JSON.parse(h['X-VectorFilter-emb']).op).toBe('cosine_within');
});
it('should set vector search headers', () => {
const h = buildHeaders({
vector_search: { column: 'emb', vector: [0.1, 0.2], metric: 'cosine', as: 'dist', direction: 'desc' },
});
expect(h['X-Vector-Search-emb']).toBe('cosine');
expect(h['X-Vector-Search-Vector']).toBe('[0.1,0.2]');
expect(h['X-Vector-Search-As']).toBe('dist');
expect(h['X-Vector-Search-Dir']).toBe('desc');
});
it('should encode X-Files as ZIP_ JSON', () => {
const xf = { tablename: 'users', prefix: 'USR', limit: 10 };
const h = buildHeaders({ xfiles: xf });
expect(h['X-Files'].startsWith('ZIP_')).toBe(true);
expect(JSON.parse(decodeHeaderValue(h['X-Files']))).toEqual(xf);
});
});
describe('buildHeaders', () => {
it('should set X-Select-Fields for columns', () => {
const h = buildHeaders({ columns: ['id', 'name', 'email'] });
@@ -126,11 +221,22 @@ describe('encodeHeaderValue / decodeHeaderValue', () => {
expect(decoded).toBe(original);
});
it('should round-trip UTF-8 values', () => {
const original = 'café ☕ 你好';
expect(decodeHeaderValue(encodeHeaderValue(original))).toBe(original);
});
it('should decode __ prefixed values', () => {
const encoded = '__' + btoa('hello');
expect(decodeHeaderValue(encoded)).toBe('hello');
});
it('should decode UTF-8 values with the __ prefix', () => {
const bytes = new TextEncoder().encode('café ☕');
const binary = Array.from(bytes, (byte) => String.fromCharCode(byte)).join('');
expect(decodeHeaderValue('__' + btoa(binary))).toBe('café ☕');
});
it('should return plain values as-is', () => {
expect(decodeHeaderValue('plain')).toBe('plain');
});
@@ -142,6 +248,7 @@ describe('HeaderSpecClient', () => {
function mockFetch<T>(data: APIResponse<T>, ok = true) {
return vi.fn().mockResolvedValue({
ok,
headers: new Headers(),
json: () => Promise.resolve(data),
});
}
+30
View File
@@ -0,0 +1,30 @@
import type { ClientConfig } from './types';
/** Merge HTTP headers case-insensitively, preserving the winning spelling. */
export function mergeHeaders(...sources: Record<string, string>[]): Record<string, string> {
const result: Record<string, string> = {};
for (const source of sources) {
for (const [name, value] of Object.entries(source)) {
for (const existing of Object.keys(result)) {
if (existing.toLowerCase() === name.toLowerCase()) delete result[existing];
}
Object.defineProperty(result, name, { value, enumerable: true, configurable: true, writable: true });
}
}
return result;
}
export function clientHeaders(config: ClientConfig): Record<string, string> {
return mergeHeaders(
{ 'Content-Type': 'application/json' },
config.headers ?? {},
config.token ? { Authorization: `Bearer ${config.token}` } : {},
);
}
export function clientCacheKey(config: ClientConfig): string {
const headers = Object.entries(clientHeaders(config))
.map(([name, value]) => [name.toLowerCase(), value])
.sort(([a], [b]) => a.localeCompare(b));
return JSON.stringify([config.baseUrl, headers]);
}
+216
View File
@@ -0,0 +1,216 @@
// Types aligned with Go pkg/common/types.go
export type Operator =
| 'eq' | 'neq' | 'gt' | 'gte' | 'lt' | 'lte'
| 'like' | 'ilike' | 'in'
| 'contains' | 'startswith' | 'endswith'
| 'between' | 'between_inclusive'
| 'is_null' | 'is_not_null'
// PostGIS spatial (sent via X-SpatialFilter-{col})
| 'st_dwithin' | 'bbox'
// pgvector similarity (sent via X-VectorFilter-{col})
| 'l2_within' | 'cosine_within' | 'ip_within';
export type Operation = 'read' | 'create' | 'update' | 'delete';
export type SortDirection = 'asc' | 'desc' | 'ASC' | 'DESC';
export interface Parameter {
name: string;
value: string;
sequence?: number;
}
export interface PreloadOption {
relation: string;
table_name?: string;
columns?: string[];
omit_columns?: string[];
sort?: SortOption[];
filters?: FilterOption[];
where?: string;
limit?: number;
offset?: number;
updatable?: boolean;
computed_ql?: Record<string, string>;
recursive?: boolean;
// Relationship keys
primary_key?: string;
related_key?: string;
foreign_key?: string;
recursive_child_key?: string;
// Custom SQL JOINs
sql_joins?: string[];
join_aliases?: string[];
}
export interface FilterOption {
column: string;
operator: Operator | string;
value: any;
logic_operator?: 'AND' | 'OR';
}
export interface SortOption {
column: string;
direction: SortDirection;
}
export interface CustomOperator {
name: string;
sql: string;
}
export interface ComputedColumn {
name: string;
expression: string;
}
export type VectorMetric = 'l2' | 'cosine' | 'ip';
export type ResponseFormat = 'simple' | 'detail' | 'syncfusion';
/** pgvector KNN search: order by distance between `column` and `vector`. */
export interface VectorSearchOption {
column: string;
vector: number[];
metric?: VectorMetric;
/** Distance column alias. Default `_distance` */
as?: string;
direction?: 'asc' | 'desc';
}
/** LEFT JOIN expansion of a relation (X-Expand). */
export interface ExpandOption {
relation: string;
columns?: string[];
}
/** X-Files configuration (Go restheadspec XFiles). Sent as a single JSON header. */
export interface XFiles {
tablename?: string;
schema?: string;
primarykey?: string;
foreignkey?: string;
relatedkey?: string;
sort?: string[];
prefix?: string;
editable?: boolean;
recursive?: boolean;
expand?: boolean;
rownumber?: boolean;
skipcount?: boolean;
offset?: number;
limit?: number;
columns?: string[];
omit_columns?: string[];
cql_columns?: string[];
sql_joins?: string[];
sql_or?: string[];
sql_and?: string[];
parenttables?: XFiles[];
childtables?: XFiles[];
filter_fields?: { field: string; value: string; operator: string }[];
cursor_forward?: string;
cursor_backward?: string;
}
export interface Options {
preload?: PreloadOption[];
columns?: string[];
omit_columns?: string[];
filters?: FilterOption[];
sort?: SortOption[];
limit?: number;
offset?: number;
customOperators?: CustomOperator[];
computedColumns?: ComputedColumn[];
parameters?: Parameter[];
cursor_forward?: string;
cursor_backward?: string;
fetch_row_number?: string;
vector_search?: VectorSearchOption;
}
/** Options only available to the header-based (restheadspec) protocol. */
export interface HeaderSpecOptions extends Options {
/** X-Expand: LEFT JOIN relations */
expand?: ExpandOption[];
/** X-Custom-SQL-Join: raw JOIN clauses */
custom_sql_joins?: string[];
/** X-Custom-SQL-Or: raw SQL, OR-combined */
custom_sql_or?: string[];
/** X-SearchCols: columns for multi-column search */
search_columns?: string[];
/** X-AdvSQL-{col}: column -> SQL expression */
advanced_sql?: Record<string, string>;
/** X-Clean-JSON */
clean_json?: boolean;
/** X-Distinct */
distinct?: boolean;
/** X-SkipCount: skip total count query */
skip_count?: boolean;
/** X-SkipCache */
skip_cache?: boolean;
/** X-PKRow: primary key value of a row to fetch */
pk_row?: string;
/** X-SimpleApi / X-DetailApi / X-Syncfusion */
response_format?: ResponseFormat;
/** X-Single-Record-As-Object (server default true) */
single_record_as_object?: boolean;
/** X-Transaction-Atomic */
atomic_transaction?: boolean;
/** X-Files: single JSON configuration */
xfiles?: XFiles;
}
export interface RequestBody {
operation: Operation;
id?: number | string | string[];
data?: any | any[];
options?: Options;
}
export interface Metadata {
total: number;
count: number;
filtered: number;
limit: number;
offset: number;
row_number?: number;
}
export interface APIError {
code: string;
message: string;
details?: any;
detail?: string;
}
export interface APIResponse<T = any> {
success: boolean;
data: T;
metadata?: Metadata;
error?: APIError;
}
export interface Column {
name: string;
type: string;
is_nullable: boolean;
is_primary: boolean;
is_unique: boolean;
has_index: boolean;
}
export interface TableMetadata {
schema: string;
table: string;
columns: Column[];
relations: string[];
}
export interface ClientConfig {
baseUrl: string;
token?: string;
/** Custom HTTP headers. Token and HeaderSpec query options take precedence. */
headers?: Record<string, string>;
}
@@ -1,9 +1,11 @@
import { clientCacheKey, clientHeaders, mergeHeaders } from '../common/http';
import { b64DecodeUnicode, b64EncodeUnicode } from '@warkypublic/artemis-kit/base64';
import type {
APIResponse,
ClientConfig,
CustomOperator,
FilterOption,
Options,
HeaderSpecOptions,
PreloadOption,
SortOption,
} from "../common/types";
@@ -12,10 +14,7 @@ import type {
* Encode a value with base64 and ZIP_ prefix for complex header values.
*/
export function encodeHeaderValue(value: string): string {
if (typeof btoa === "function") {
return "ZIP_" + btoa(value);
}
return "ZIP_" + Buffer.from(value, "utf-8").toString("base64");
return "ZIP_" + b64EncodeUnicode(value);
}
/**
@@ -41,10 +40,7 @@ export function decodeHeaderValue(value: string): string {
}
function decodeBase64(str: string): string {
if (typeof atob === "function") {
return atob(str);
}
return Buffer.from(str, "base64").toString("utf-8");
return b64DecodeUnicode(str);
}
/**
@@ -63,8 +59,15 @@ function decodeBase64(str: string): string {
* - X-Fetch-RowNumber: row number fetch
* - X-CQL-SEL-{col}: computed columns
* - X-Custom-SQL-W: custom operators (AND)
* - X-Preload-Where: where for X-Preload (extra where groups use X-Preload-{n}[-Where])
* - X-SpatialFilter-{col} / X-VectorFilter-{col}: JSON {op,value,logic}
* - X-Vector-Search-{col|vector|as|dir}: pgvector KNN
* - X-Expand, X-Custom-SQL-Join, X-Custom-SQL-Or, X-SearchCols, X-AdvSQL-{col}
* - X-Clean-JSON, X-Distinct, X-SkipCount, X-SkipCache, X-PKRow
* - X-SimpleApi / X-DetailApi / X-Syncfusion, X-Single-Record-As-Object
* - X-Transaction-Atomic, X-Files
*/
export function buildHeaders(options: Options): Record<string, string> {
export function buildHeaders(options: HeaderSpecOptions): Record<string, string> {
const headers: Record<string, string> = {};
// Column selection
@@ -83,6 +86,17 @@ export function buildHeaders(options: Options): Record<string, string> {
const op = mapOperatorToHeaderOp(filter.operator);
const valueStr = formatFilterValue(filter);
const geoPrefix = geoFilterHeader(filter.operator);
if (geoPrefix) {
const payload: Record<string, unknown> = {
op: filter.operator,
value: filter.value,
};
if (logicOp === "OR") payload.logic = "or";
headers[`${geoPrefix}${filter.column}`] = JSON.stringify(payload);
continue;
}
if (filter.operator === "eq" && logicOp === "AND") {
// Simple field filter shorthand
headers[`X-FieldFilter-${filter.column}`] = valueStr;
@@ -121,13 +135,94 @@ export function buildHeaders(options: Options): Record<string, string> {
// Preload
if (options.preload?.length) {
const parts = options.preload.map((p: PreloadOption) => {
if (p.columns?.length) {
return `${p.relation}:${p.columns.join(",")}`;
// Go applies X-Preload-Where to every preload in the matching X-Preload header,
// so preloads are grouped by where clause.
const groups = new Map<string, string[]>();
for (const p of options.preload) {
const spec = p.columns?.length
? `${p.relation}:${p.columns.join(",")}`
: p.relation;
const where = p.where ?? "";
groups.set(where, [...(groups.get(where) ?? []), spec]);
}
let n = 0;
for (const [where, specs] of groups) {
if (!where) {
headers["X-Preload"] = specs.join("|");
} else if (!groups.has("") && n === 0) {
// X-Preload-Where would also apply to a where-less X-Preload, so only use it alone
headers["X-Preload"] = specs.join("|");
headers["X-Preload-Where"] = where;
n++;
} else {
n++;
headers[`X-Preload-${n}`] = specs.join("|");
headers[`X-Preload-${n}-Where`] = where;
}
return p.relation;
});
headers["X-Preload"] = parts.join("|");
}
}
// Expand (LEFT JOIN)
if (options.expand?.length) {
headers["X-Expand"] = options.expand
.map((e) =>
e.columns?.length ? `${e.relation}:${e.columns.join(",")}` : e.relation,
)
.join("|");
}
if (options.custom_sql_joins?.length) {
headers["X-Custom-SQL-Join"] = options.custom_sql_joins.join("|");
}
if (options.custom_sql_or?.length) {
headers["X-Custom-SQL-Or"] = options.custom_sql_or.join(" OR ");
}
if (options.search_columns?.length) {
headers["X-SearchCols"] = options.search_columns.join(",");
}
if (options.advanced_sql) {
for (const [col, sql] of Object.entries(options.advanced_sql)) {
headers[`X-AdvSQL-${col}`] = sql;
}
}
// pgvector KNN search
if (options.vector_search) {
const vs = options.vector_search;
headers[`X-Vector-Search-${vs.column}`] = vs.metric ?? "l2";
headers["X-Vector-Search-Vector"] = JSON.stringify(vs.vector);
if (vs.as) headers["X-Vector-Search-As"] = vs.as;
if (vs.direction) headers["X-Vector-Search-Dir"] = vs.direction;
}
// Flags
const flags: [string, boolean | undefined][] = [
["X-Clean-JSON", options.clean_json],
["X-Distinct", options.distinct],
["X-SkipCount", options.skip_count],
["X-SkipCache", options.skip_cache],
["X-Transaction-Atomic", options.atomic_transaction],
["X-Single-Record-As-Object", options.single_record_as_object],
];
for (const [name, val] of flags) {
if (val !== undefined) headers[name] = String(val);
}
if (options.pk_row) {
headers["X-PKRow"] = options.pk_row;
}
if (options.response_format) {
const formatHeaders = {
simple: "X-SimpleApi",
detail: "X-DetailApi",
syncfusion: "X-Syncfusion",
} as const;
headers[formatHeaders[options.response_format]] = "true";
}
if (options.xfiles) {
headers["X-Files"] = encodeHeaderValue(JSON.stringify(options.xfiles));
}
// Fetch row number
@@ -153,6 +248,17 @@ export function buildHeaders(options: Options): Record<string, string> {
return headers;
}
const VECTOR_OPS = new Set(["l2_within", "cosine_within", "ip_within"]);
function geoFilterHeader(operator: string): string | null {
const op = operator.toLowerCase();
if (VECTOR_OPS.has(op) || op.endsWith("_within")) return "X-VectorFilter-";
if (op.startsWith("st_") || op === "bbox" || op === "&&") {
return "X-SpatialFilter-";
}
return null;
}
function mapOperatorToHeaderOp(operator: string): string {
switch (operator) {
case "eq":
@@ -203,7 +309,7 @@ function formatFilterValue(filter: FilterOption): string {
const instances = new Map<string, HeaderSpecClient>();
export function getHeaderSpecClient(config: ClientConfig): HeaderSpecClient {
const key = config.baseUrl;
const key = clientCacheKey(config);
let instance = instances.get(key);
if (!instance) {
instance = new HeaderSpecClient(config);
@@ -222,7 +328,7 @@ export class HeaderSpecClient {
private config: ClientConfig;
constructor(config: ClientConfig) {
this.config = config;
this.config = { ...config, headers: { ...config.headers } };
}
private buildUrl(schema: string, entity: string, id?: string): string {
@@ -234,13 +340,7 @@ export class HeaderSpecClient {
}
private baseHeaders(): Record<string, string> {
const headers: Record<string, string> = {
"Content-Type": "application/json",
};
if (this.config.token) {
headers["Authorization"] = `Bearer ${this.config.token}`;
}
return headers;
return clientHeaders(this.config);
}
private async fetchWithError<T>(
@@ -290,13 +390,13 @@ export class HeaderSpecClient {
schema: string,
entity: string,
id?: string,
options?: Options,
options?: HeaderSpecOptions,
): Promise<APIResponse<T>> {
const url = this.buildUrl(schema, entity, id);
const optHeaders = options ? buildHeaders(options) : {};
return this.fetchWithError<T>(url, {
method: "GET",
headers: { ...this.baseHeaders(), ...optHeaders },
headers: mergeHeaders(this.baseHeaders(), optHeaders),
});
}
@@ -304,13 +404,13 @@ export class HeaderSpecClient {
schema: string,
entity: string,
data: any,
options?: Options,
options?: HeaderSpecOptions,
): Promise<APIResponse<T>> {
const url = this.buildUrl(schema, entity);
const optHeaders = options ? buildHeaders(options) : {};
return this.fetchWithError<T>(url, {
method: "POST",
headers: { ...this.baseHeaders(), ...optHeaders },
headers: mergeHeaders(this.baseHeaders(), optHeaders),
body: JSON.stringify(data),
});
}
@@ -320,13 +420,13 @@ export class HeaderSpecClient {
entity: string,
id: string,
data: any,
options?: Options,
options?: HeaderSpecOptions,
): Promise<APIResponse<T>> {
const url = this.buildUrl(schema, entity, id);
const optHeaders = options ? buildHeaders(options) : {};
return this.fetchWithError<T>(url, {
method: "PUT",
headers: { ...this.baseHeaders(), ...optHeaders },
headers: mergeHeaders(this.baseHeaders(), optHeaders),
body: JSON.stringify(data),
});
}
@@ -1,9 +1,10 @@
import { clientCacheKey, clientHeaders } from '../common/http';
import type { ClientConfig, APIResponse, TableMetadata, Options, RequestBody } from '../common/types';
const instances = new Map<string, ResolveSpecClient>();
export function getResolveSpecClient(config: ClientConfig): ResolveSpecClient {
const key = config.baseUrl;
const key = clientCacheKey(config);
let instance = instances.get(key);
if (!instance) {
instance = new ResolveSpecClient(config);
@@ -16,7 +17,7 @@ export class ResolveSpecClient {
private config: ClientConfig;
constructor(config: ClientConfig) {
this.config = config;
this.config = { ...config, headers: { ...config.headers } };
}
private buildUrl(schema: string, entity: string, id?: string): string {
@@ -28,15 +29,7 @@ export class ResolveSpecClient {
}
private baseHeaders(): HeadersInit {
const headers: Record<string, string> = {
'Content-Type': 'application/json',
};
if (this.config.token) {
headers['Authorization'] = `Bearer ${this.config.token}`;
}
return headers;
return clientHeaders(this.config);
}
private async fetchWithError<T>(url: string, options: RequestInit): Promise<APIResponse<T>> {
@@ -14,7 +14,7 @@ export default defineConfig({
fileName: (format) => `index.${format === 'es' ? 'js' : 'cjs'}`,
},
rollupOptions: {
external: ['uuid', 'semver'],
external: ['uuid', 'semver', '@warkypublic/artemis-kit/base64'],
},
},
});
+6
View File
@@ -0,0 +1,6 @@
__pycache__/
*.egg-info/
.venv/
dist/
.pytest_cache/
.coverage
+142
View File
@@ -0,0 +1,142 @@
# resolvespec (Python)
Python client for ResolveSpec REST, HeaderSpec (restheadspec), FunctionSpec and WebSocketSpec. Port of `resolvespec-js`.
- Python >= 3.11, `httpx` (REST, sync + async), `websockets` (WS, async)
- Options/filters/sorts are plain dicts using the wire key names (`TypedDict` hints in `resolvespec.types`)
```
pip install resolvespec
```
## Clients
| Protocol | Sync | Async | Transport |
|---|---|---|---|
| ResolveSpec | `ResolveSpecClient` | `AsyncResolveSpecClient` | POST + JSON body `{operation, id, data, options}` |
| HeaderSpec | `HeaderSpecClient` | `AsyncHeaderSpecClient` | GET/POST/PUT/DELETE, options as `X-*` headers |
| FunctionSpec | `FuncSpecClient` | `AsyncFuncSpecClient` | user-defined SQL endpoints; params via query string + `X-*` headers |
| WebSocketSpec | - | `WebSocketClient` | WebSocket JSON messages |
Constructor (REST): `Client(base_url, token=None, headers=None, timeout=30.0)`
- `token` -> `Authorization: Bearer`; wins over `headers`
- `headers`: custom headers, merged case-insensitively; snapshot at construction
- Sync: context manager / `close()`. Async: `async with` / `await aclose()`
- Cached sync factories: `get_resolvespec_client()`, `get_headerspec_client()` (same args -> same instance)
## ResolveSpec
URL: `{base}/{schema}/{entity}[/{id}]`
| Method | Signature |
|---|---|
| `get_metadata` | `(schema, entity)` (GET) |
| `read` | `(schema, entity, id=None, options=None)` |
| `create` | `(schema, entity, data, options=None)` |
| `update` | `(schema, entity, data, id=None, options=None)` |
| `delete` | `(schema, entity, id)` |
`id`: int/str -> URL path; `list[str]` -> body `id`.
Returns `{"success", "data", "metadata"?, "error"?}`.
## HeaderSpec
| Method | HTTP | Signature |
|---|---|---|
| `read` | GET | `(schema, entity, id=None, options=None)` |
| `create` | POST | `(schema, entity, data, options=None)` |
| `update` | PUT | `(schema, entity, id, data, options=None)` |
| `delete` | DELETE | `(schema, entity, id)` |
Response metadata derived from `Content-Range` (`offset-end/total`) and `X-Limit`.
`build_headers(options)`, `encode_header_value()` / `decode_header_value()` (`ZIP_` / `__` base64) are exported.
### Option -> header
| Option | Header |
|---|---|
| `columns` / `omit_columns` | `X-Select-Fields` / `X-Not-Select-Fields` |
| filter `eq` + AND | `X-FieldFilter-{col}` |
| filter AND / OR | `X-SearchOp-{op}-{col}` / `X-SearchOr-{op}-{col}` |
| spatial (`st_*`, `bbox`) / vector (`*_within`) filter | `X-SpatialFilter-{col}` / `X-VectorFilter-{col}` (JSON) |
| `sort` | `X-Sort` (`+col,-col`) |
| `limit` / `offset` | `X-Limit` / `X-Offset` |
| `cursor_forward` / `cursor_backward` | `X-Cursor-Forward` / `X-Cursor-Backward` |
| `preload` | `X-Preload` (`Rel:c1,c2\|Rel2`), `X-Preload-Where`, `X-Preload-{n}[-Where]` |
| `expand` | `X-Expand` |
| `custom_sql_joins` / `custom_sql_or` | `X-Custom-SQL-Join` / `X-Custom-SQL-Or` |
| `search_columns` | `X-SearchCols` |
| `advanced_sql` | `X-AdvSQL-{col}` |
| `computedColumns` | `X-CQL-SEL-{name}` |
| `customOperators` | `X-Custom-SQL-W` (AND-joined) |
| `vector_search` | `X-Vector-Search-{col}`, `-Vector`, `-As`, `-Dir` |
| `fetch_row_number` | `X-Fetch-RowNumber` |
| `clean_json`, `distinct`, `skip_count`, `skip_cache`, `atomic_transaction`, `single_record_as_object` | `X-Clean-JSON`, `X-Distinct`, `X-SkipCount`, `X-SkipCache`, `X-Transaction-Atomic`, `X-Single-Record-As-Object` |
| `pk_row` | `X-PKRow` |
| `response_format` (`simple`/`detail`/`syncfusion`) | `X-SimpleApi` / `X-DetailApi` / `X-Syncfusion` |
| `xfiles` | `X-Files` (`ZIP_` base64 JSON) |
Filter operator -> header op: `eq equals`, `neq notequals`, `gt greaterthan`, `gte greaterthanorequal`, `lt lessthan`, `lte lessthanorequal`, `like/ilike/contains contains`, `startswith beginswith`, `endswith`, `in`, `between`, `between_inclusive betweeninclusive`, `is_null empty`, `is_not_null notempty`.
## FunctionSpec
Routes are defined by the server app, so calls take a `path`. The server never reads a request body.
| Method | Server handler | Result |
|---|---|---|
| `query(path, params=None, options=None, *, method="GET")` | `SqlQuery` (single record) | `{success, data}` |
| `query_list(path, params=None, options=None, *, method="GET")` | `SqlQueryList` | `{success, data, metadata}` (from `Content-Range: items a-b/total`) |
- `params` -> query string. `bool` -> `true/false`, `None` skipped, `list` -> repeated key (server: `IN` filter). `p-` prefixed names are substituted into the SQL.
- `options` -> `X-*` headers. Query values override headers of the same name.
- 206 Partial Content (more rows than returned) is treated as success.
| Option | Header |
|---|---|
| `filters` (`eq`+AND) | `X-FieldFilter-{col}` |
| `filters` (other) | `X-SearchOp-{op}-{col}` / `X-SearchOr-{op}-{col}` |
| `search_filters` `{col: text}` | `X-SearchFilter-{col}` (ILIKE) |
| `custom_sql_where` / `custom_sql_or` | `X-Custom-SQL-W` / `X-Custom-SQL-Or` |
| `sort` | `X-Sort` as SQL terms: `col ASC,col DESC` |
| `limit` / `offset` | `X-Limit` / `X-Offset` |
| `distinct`, `skip_count`, `skip_cache` | `X-Distinct`, `X-SkipCount`, `X-SkipCache` |
| `response_format` | `X-SimpleApi` / `X-DetailApi` / `X-Syncfusion` (`data` shape changes: array / `{items,...}` / `{result,count}`) |
Server limits:
- `sort` goes verbatim into `ORDER BY`; `-col` (restheadspec style) does **not** mean DESC.
- `X-Select-Fields` / `X-Not-Select-Fields` are no-ops server-side, so not exposed.
- One search operator per column; same column twice keeps the last.
- Values starting with `ZIP_` / `__` are base64-decoded by the server; such plaintext cannot be sent.
- Non-ASCII / control-char values are sent `ZIP_`-encoded automatically.
## WebSocketSpec
`WebSocketClient(url, *, reconnect=True, reconnect_interval=3.0, max_reconnect_attempts=10, heartbeat_interval=30.0, request_timeout=30.0, subscribe_timeout=10.0, headers=None)`
| Method | Notes |
|---|---|
| `connect()` / `close()` | also `async with` |
| `request(operation, entity, *, schema, record_id, data, options)` | returns response `data` |
| `read(entity, *, schema, record_id, filters, columns, sort, preload, limit, offset)` | |
| `create(entity, data, *, schema)` | |
| `update(entity, id, data, *, schema)` | |
| `delete(entity, id, *, schema)` | |
| `meta(entity, *, schema)` | |
| `subscribe(entity, callback, *, schema, filters)` | returns subscription id; callback gets notification dict (sync or async) |
| `unsubscribe(subscription_id)` | |
| `on(event, cb)` / `off(event)` | events: `connect`, `disconnect`, `error`, `message`, `state_change` |
| `state`, `is_connected()`, `get_subscriptions()` | |
Auto-reconnect does not restore subscriptions; re-subscribe on `connect`.
## Errors
`ResolveSpecError(message, status_code, code, details)` on non-2xx (REST) or failed response / timeout / not connected (WS).
## Dev
```
pip install -e '.[dev]'
pytest
```
+24
View File
@@ -0,0 +1,24 @@
[build-system]
requires = ["hatchling"]
build-backend = "hatchling.build"
[project]
name = "resolvespec"
version = "1.0.0"
description = "Python client for ResolveSpec REST, HeaderSpec and WebSocket APIs"
readme = "README.md"
requires-python = ">=3.11"
license = { text = "MIT" }
authors = [{ name = "Hein (Warkanum) Puth" }]
keywords = ["resolvespec", "headerspec", "websocket", "rest-client", "api-client"]
dependencies = ["httpx>=0.27", "websockets>=13"]
[project.optional-dependencies]
dev = ["pytest>=8", "pytest-asyncio>=0.23", "pytest-cov"]
[tool.hatch.build.targets.wheel]
packages = ["src/resolvespec"]
[tool.pytest.ini_options]
testpaths = ["tests"]
asyncio_mode = "auto"
@@ -0,0 +1,43 @@
"""ResolveSpec Python client: REST (ResolveSpec), HeaderSpec and WebSocketSpec."""
from typing import Mapping, Optional
from .headerspec import (
AsyncHeaderSpecClient,
HeaderSpecClient,
build_headers,
decode_header_value,
encode_header_value,
)
from .funcspec import AsyncFuncSpecClient, FuncSpecClient
from .http import ResolveSpecError, merge_headers
from .resolvespec import AsyncResolveSpecClient, ResolveSpecClient
from .types import * # noqa: F401,F403
from .websocket import Subscription, WebSocketClient
def _cache_key(base_url: str, token: Optional[str], headers: Optional[Mapping[str, str]]):
return (
base_url,
token,
tuple(sorted((k.lower(), v) for k, v in (headers or {}).items())),
)
_resolvespec: dict = {}
_headerspec: dict = {}
def get_resolvespec_client(base_url: str, token: Optional[str] = None, headers: Optional[Mapping[str, str]] = None) -> ResolveSpecClient:
"""Cached sync client, keyed by base_url + token + headers (case-insensitive names)."""
key = _cache_key(base_url, token, headers)
if key not in _resolvespec:
_resolvespec[key] = ResolveSpecClient(base_url, token, headers)
return _resolvespec[key]
def get_headerspec_client(base_url: str, token: Optional[str] = None, headers: Optional[Mapping[str, str]] = None) -> HeaderSpecClient:
"""Cached sync client, keyed by base_url + token + headers (case-insensitive names)."""
key = _cache_key(base_url, token, headers)
if key not in _headerspec:
_headerspec[key] = HeaderSpecClient(base_url, token, headers)
return _headerspec[key]
@@ -0,0 +1,197 @@
"""FunctionSpec client: calls user-defined SQL endpoints (Go pkg/funcspec).
Routes are defined by the server application, so calls take a `path`.
Parameters are sent as query string values and/or `X-*` headers; the server never
reads a request body. Query-string values override headers of the same name.
Server behaviour worth knowing (pkg/funcspec):
- `sort` is inserted raw into ORDER BY, so it must be SQL (`col DESC`), not `-col`.
- Field selection (`X-Select-Fields`) is a no-op server-side, so it is not exposed.
- Only one search operator per column is kept.
- Values starting with `ZIP_` or `__` are base64-decoded by the server (even after our
own encoding), so such plaintext values cannot be sent faithfully.
"""
from __future__ import annotations
import re
from typing import Any, Dict, List, Mapping, Optional
import httpx
from .headerspec import _OPERATOR_MAP, _bool, _filter_value, encode_header_value
from .http import client_headers, error_from, merge_headers, parse_json
from .types import APIResponse, FuncSpecOptions
Params = Mapping[str, Any]
_CONTENT_RANGE = re.compile(r"(\d+)-(\d+)/(\d+)")
def _safe(value: str) -> str:
"""Encode values that are unsafe as raw header/query text (non-ASCII, control chars, edge spaces)."""
if not value.isascii() or not value.isprintable() or value != value.strip():
return encode_header_value(value)
return value
def build_headers(options: Mapping[str, Any]) -> Dict[str, str]:
"""Build the X-* headers understood by funcspec.ParseParameters."""
h: Dict[str, str] = {}
o = options
for f in o.get("filters") or []:
operator = f["operator"]
logic = f.get("logic_operator") or "AND"
value = _safe(_filter_value(f))
if operator == "eq" and logic == "AND":
h[f"X-FieldFilter-{f['column']}"] = value
else:
kind = "X-SearchOr" if logic == "OR" else "X-SearchOp"
h[f"{kind}-{_OPERATOR_MAP.get(operator, operator)}-{f['column']}"] = value
for col, text in (o.get("search_filters") or {}).items():
h[f"X-SearchFilter-{col}"] = _safe(str(text)) # CAST(col AS TEXT) ILIKE %text%
if o.get("custom_sql_where"):
h["X-Custom-SQL-W"] = _safe(o["custom_sql_where"])
if o.get("custom_sql_or"):
h["X-Custom-SQL-Or"] = _safe(o["custom_sql_or"])
if o.get("sort"):
h["X-Sort"] = _safe(",".join(_sort_term(s) for s in o["sort"]))
if o.get("limit") is not None:
h["X-Limit"] = str(o["limit"])
if o.get("offset") is not None:
h["X-Offset"] = str(o["offset"])
for name, key in (("X-Distinct", "distinct"), ("X-SkipCount", "skip_count"), ("X-SkipCache", "skip_cache")):
if o.get(key) is not None:
h[name] = _bool(o[key])
fmt = o.get("response_format")
if fmt:
h[{"simple": "X-SimpleApi", "detail": "X-DetailApi", "syncfusion": "X-Syncfusion"}[fmt]] = "true"
return h
def _sort_term(s: Mapping[str, str]) -> str:
# funcspec puts this verbatim into ORDER BY
return f"{s['column']} {'DESC' if s.get('direction', 'asc').upper() == 'DESC' else 'ASC'}"
def build_query(params: Optional[Params]) -> Dict[str, Any]:
"""Query-string values: bools -> true/false, lists -> repeated keys (server: IN filter)."""
out: Dict[str, Any] = {}
for k, v in (params or {}).items():
if v is None:
continue
if isinstance(v, (list, tuple)):
out[k] = [_safe(_q(x)) for x in v]
else:
out[k] = _safe(_q(v))
return out
def _q(v: Any) -> str:
return _bool(v) if isinstance(v, bool) else str(v)
def _metadata(response: httpx.Response, options: Optional[Mapping[str, Any]]) -> Dict[str, int]:
"""Content-Range is `items {offset}-{offset+len}/{total}`."""
m = _CONTENT_RANGE.search(response.headers.get("content-range", ""))
start, end, total = (int(x) for x in m.groups()) if m else (0, 0, 0)
return {
"total": total,
"count": end - start,
"filtered": total,
"offset": start,
"limit": int((options or {}).get("limit") or 0),
}
def _wrap(response: httpx.Response, options: Optional[Mapping[str, Any]], with_metadata: bool) -> APIResponse:
data = parse_json(response)
if not response.is_success: # 206 Partial Content is success
raise error_from(response, data)
result: APIResponse = {"success": True, "data": data}
if with_metadata:
result["metadata"] = _metadata(response, options)
return result
class _Base:
def __init__(
self,
base_url: str,
token: Optional[str] = None,
headers: Optional[Mapping[str, str]] = None,
timeout: Optional[float] = 30.0,
):
self.base_url = base_url
self.token = token
self.headers = dict(headers or {}) # snapshot
self.timeout = timeout
def _req(self, method: str, path: str, params: Optional[Params], options: Optional[Mapping[str, Any]]):
url = f"{self.base_url.rstrip('/')}/{path.lstrip('/')}"
headers = merge_headers(
client_headers(self.token, self.headers),
build_headers(options) if options else {},
)
return method.upper(), url, headers, build_query(params)
class FuncSpecClient(_Base):
"""Synchronous client. Use as a context manager or call close()."""
def __init__(self, *args: Any, transport: Optional[httpx.BaseTransport] = None, **kwargs: Any):
super().__init__(*args, **kwargs)
self._http = httpx.Client(timeout=self.timeout, transport=transport)
def close(self) -> None:
self._http.close()
def __enter__(self) -> "FuncSpecClient":
return self
def __exit__(self, *exc: Any) -> None:
self.close()
def _send(self, req, options, with_metadata) -> APIResponse:
method, url, headers, query = req
return _wrap(self._http.request(method, url, headers=headers, params=query), options, with_metadata)
def query(self, path: str, params: Optional[Params] = None, options: Optional[FuncSpecOptions] = None, *, method: str = "GET") -> APIResponse:
"""Single-record endpoint (Handler.SqlQuery). `data` is the row object."""
return self._send(self._req(method, path, params, options), options, False)
def query_list(self, path: str, params: Optional[Params] = None, options: Optional[FuncSpecOptions] = None, *, method: str = "GET") -> APIResponse:
"""List endpoint (Handler.SqlQueryList). Adds `metadata` from Content-Range."""
return self._send(self._req(method, path, params, options), options, True)
class AsyncFuncSpecClient(_Base):
"""Asyncio client. Use as an async context manager or await aclose()."""
def __init__(self, *args: Any, transport: Optional[httpx.AsyncBaseTransport] = None, **kwargs: Any):
super().__init__(*args, **kwargs)
self._http = httpx.AsyncClient(timeout=self.timeout, transport=transport)
async def aclose(self) -> None:
await self._http.aclose()
async def __aenter__(self) -> "AsyncFuncSpecClient":
return self
async def __aexit__(self, *exc: Any) -> None:
await self.aclose()
async def _send(self, req, options, with_metadata) -> APIResponse:
method, url, headers, query = req
return _wrap(await self._http.request(method, url, headers=headers, params=query), options, with_metadata)
async def query(self, path: str, params: Optional[Params] = None, options: Optional[FuncSpecOptions] = None, *, method: str = "GET") -> APIResponse:
return await self._send(self._req(method, path, params, options), options, False)
async def query_list(self, path: str, params: Optional[Params] = None, options: Optional[FuncSpecOptions] = None, *, method: str = "GET") -> APIResponse:
return await self._send(self._req(method, path, params, options), options, True)
@@ -0,0 +1,336 @@
"""HeaderSpec client: query options sent as HTTP headers (Go restheadspec).
Methods: GET=read, POST=create, PUT=update, DELETE=delete.
"""
from __future__ import annotations
import base64
import json
import re
from typing import Any, Dict, Mapping, Optional
import httpx
from .http import build_url, client_headers, error_from, merge_headers, parse_json
from .types import APIResponse, FilterOption, HeaderSpecOptions
_PREFIXES = ("ZIP_", "__")
_OPERATOR_MAP = {
"eq": "equals",
"neq": "notequals",
"gt": "greaterthan",
"gte": "greaterthanorequal",
"lt": "lessthan",
"lte": "lessthanorequal",
"like": "contains",
"ilike": "contains",
"contains": "contains",
"startswith": "beginswith",
"endswith": "endswith",
"in": "in",
"between": "between",
"between_inclusive": "betweeninclusive",
"is_null": "empty",
"is_not_null": "notempty",
}
def encode_header_value(value: str) -> str:
"""Base64 (UTF-8) with ZIP_ prefix, for complex header values."""
return "ZIP_" + base64.b64encode(value.encode("utf-8")).decode("ascii")
def decode_header_value(value: str) -> str:
"""Decode a value that may carry a ZIP_ or __ base64 prefix (nested allowed)."""
code = value
for prefix in _PREFIXES:
if code.startswith(prefix):
b64 = re.sub(r"[\n\r ]", "", code[len(prefix):])
b64 += "=" * (-len(b64) % 4)
code = base64.b64decode(b64).decode("utf-8")
break
if code.startswith(_PREFIXES):
code = decode_header_value(code)
return code
def _geo_header(operator: str) -> Optional[str]:
op = operator.lower()
if op.endswith("_within"):
return "X-VectorFilter-"
if op.startswith("st_") or op in ("bbox", "&&"):
return "X-SpatialFilter-"
return None
def _filter_value(f: FilterOption) -> str:
v = f.get("value")
if v is None:
return ""
if isinstance(v, (list, tuple)):
return ",".join(_scalar(x) for x in v)
return _scalar(v)
def _scalar(v: Any) -> str:
if isinstance(v, bool): # match JS String(true)
return "true" if v else "false"
return str(v)
def _bool(v: bool) -> str:
return "true" if v else "false"
def _preload_spec(p: Mapping[str, Any]) -> str:
cols = p.get("columns")
return f"{p['relation']}:{','.join(cols)}" if cols else p["relation"]
def build_headers(options: HeaderSpecOptions) -> Dict[str, str]:
"""Build restheadspec HTTP headers from options. See README for the mapping."""
h: Dict[str, str] = {}
o = options
if o.get("columns"):
h["X-Select-Fields"] = ",".join(o["columns"])
if o.get("omit_columns"):
h["X-Not-Select-Fields"] = ",".join(o["omit_columns"])
for f in o.get("filters") or []:
logic = f.get("logic_operator") or "AND"
operator = f["operator"]
op = _OPERATOR_MAP.get(operator, operator)
value = _filter_value(f)
geo = _geo_header(operator)
if geo:
payload: Dict[str, Any] = {"op": operator, "value": f.get("value")}
if logic == "OR":
payload["logic"] = "or"
h[f"{geo}{f['column']}"] = json.dumps(payload, separators=(",", ":"))
elif operator == "eq" and logic == "AND":
h[f"X-FieldFilter-{f['column']}"] = value
elif logic == "OR":
h[f"X-SearchOr-{op}-{f['column']}"] = value
else:
h[f"X-SearchOp-{op}-{f['column']}"] = value
if o.get("sort"):
h["X-Sort"] = ",".join(
("-" if s["direction"].upper() == "DESC" else "+") + s["column"] for s in o["sort"]
)
if o.get("limit") is not None:
h["X-Limit"] = str(o["limit"])
if o.get("offset") is not None:
h["X-Offset"] = str(o["offset"])
if o.get("cursor_forward"):
h["X-Cursor-Forward"] = o["cursor_forward"]
if o.get("cursor_backward"):
h["X-Cursor-Backward"] = o["cursor_backward"]
if o.get("preload"):
# Go applies X-Preload-Where to every preload in the matching X-Preload header,
# so preloads are grouped by where clause.
groups: Dict[str, list] = {}
for p in o["preload"]:
groups.setdefault(p.get("where") or "", []).append(_preload_spec(p))
n = 0
for where, specs in groups.items():
if not where:
h["X-Preload"] = "|".join(specs)
elif "" not in groups and n == 0:
# X-Preload-Where would also apply to a where-less X-Preload, so only use it alone
h["X-Preload"] = "|".join(specs)
h["X-Preload-Where"] = where
n += 1
else:
n += 1
h[f"X-Preload-{n}"] = "|".join(specs)
h[f"X-Preload-{n}-Where"] = where
if o.get("expand"):
h["X-Expand"] = "|".join(_preload_spec(e) for e in o["expand"])
if o.get("custom_sql_joins"):
h["X-Custom-SQL-Join"] = "|".join(o["custom_sql_joins"])
if o.get("custom_sql_or"):
h["X-Custom-SQL-Or"] = " OR ".join(o["custom_sql_or"])
if o.get("search_columns"):
h["X-SearchCols"] = ",".join(o["search_columns"])
for col, sql in (o.get("advanced_sql") or {}).items():
h[f"X-AdvSQL-{col}"] = sql
vs = o.get("vector_search")
if vs:
h[f"X-Vector-Search-{vs['column']}"] = vs.get("metric") or "l2"
h["X-Vector-Search-Vector"] = json.dumps(vs["vector"], separators=(",", ":"))
if vs.get("as"):
h["X-Vector-Search-As"] = vs["as"]
if vs.get("direction"):
h["X-Vector-Search-Dir"] = vs["direction"]
for name, key in (
("X-Clean-JSON", "clean_json"),
("X-Distinct", "distinct"),
("X-SkipCount", "skip_count"),
("X-SkipCache", "skip_cache"),
("X-Transaction-Atomic", "atomic_transaction"),
("X-Single-Record-As-Object", "single_record_as_object"),
):
if o.get(key) is not None:
h[name] = _bool(o[key])
if o.get("pk_row"):
h["X-PKRow"] = o["pk_row"]
fmt = o.get("response_format")
if fmt:
h[{"simple": "X-SimpleApi", "detail": "X-DetailApi", "syncfusion": "X-Syncfusion"}[fmt]] = "true"
if o.get("xfiles"):
h["X-Files"] = encode_header_value(json.dumps(o["xfiles"], separators=(",", ":")))
if o.get("fetch_row_number"):
h["X-Fetch-RowNumber"] = o["fetch_row_number"]
for cc in o.get("computedColumns") or []:
h[f"X-CQL-SEL-{cc['name']}"] = cc["expression"]
if o.get("customOperators"):
h["X-Custom-SQL-W"] = " AND ".join(co["sql"] for co in o["customOperators"])
return h
def _int(s: Optional[str]) -> int:
try:
return int(s) # type: ignore[arg-type]
except (TypeError, ValueError):
return 0
def _wrap(response: httpx.Response) -> APIResponse:
"""Wrap a raw restheadspec body, deriving metadata from Content-Range / X-Limit."""
data = parse_json(response)
if not response.is_success:
raise error_from(response, data)
cr = response.headers.get("content-range")
total = _int(cr.split("/")[-1]) if cr else 0
offset = _int(cr.split("/")[0].split("-")[0].split(" ")[-1]) if cr else 0
return {
"data": data,
"success": True,
"error": data.get("error") if isinstance(data, dict) else None,
"metadata": {
"count": total,
"total": total,
"filtered": total,
"offset": offset,
"limit": _int(response.headers.get("x-limit")),
},
}
class _Base:
def __init__(
self,
base_url: str,
token: Optional[str] = None,
headers: Optional[Mapping[str, str]] = None,
timeout: Optional[float] = 30.0,
):
self.base_url = base_url
self.token = token
self.headers = dict(headers or {}) # snapshot
self.timeout = timeout
def _base_headers(self) -> Dict[str, str]:
return client_headers(self.token, self.headers)
def _req(self, method, schema, entity, id, options=None, body=None):
opt = build_headers(options) if options else {}
return (
method,
build_url(self.base_url, schema, entity, id),
merge_headers(self._base_headers(), opt),
body,
)
def _read_req(self, schema, entity, id, options):
return self._req("GET", schema, entity, id, options)
def _create_req(self, schema, entity, data, options):
return self._req("POST", schema, entity, None, options, data)
def _update_req(self, schema, entity, id, data, options):
return self._req("PUT", schema, entity, id, options, data)
def _delete_req(self, schema, entity, id):
return self._req("DELETE", schema, entity, id)
class HeaderSpecClient(_Base):
"""Synchronous client. Use as a context manager or call close()."""
def __init__(self, *args: Any, transport: Optional[httpx.BaseTransport] = None, **kwargs: Any):
super().__init__(*args, **kwargs)
self._http = httpx.Client(timeout=self.timeout, transport=transport)
def close(self) -> None:
self._http.close()
def __enter__(self) -> "HeaderSpecClient":
return self
def __exit__(self, *exc: Any) -> None:
self.close()
def _send(self, req) -> APIResponse:
method, url, headers, body = req
return _wrap(self._http.request(method, url, headers=headers, json=body))
def read(self, schema: str, entity: str, id: Optional[str] = None, options: Optional[HeaderSpecOptions] = None) -> APIResponse:
return self._send(self._read_req(schema, entity, id, options))
def create(self, schema: str, entity: str, data: Any, options: Optional[HeaderSpecOptions] = None) -> APIResponse:
return self._send(self._create_req(schema, entity, data, options))
def update(self, schema: str, entity: str, id: str, data: Any, options: Optional[HeaderSpecOptions] = None) -> APIResponse:
return self._send(self._update_req(schema, entity, id, data, options))
def delete(self, schema: str, entity: str, id: str) -> APIResponse:
return self._send(self._delete_req(schema, entity, id))
class AsyncHeaderSpecClient(_Base):
"""Asyncio client. Use as an async context manager or await aclose()."""
def __init__(self, *args: Any, transport: Optional[httpx.AsyncBaseTransport] = None, **kwargs: Any):
super().__init__(*args, **kwargs)
self._http = httpx.AsyncClient(timeout=self.timeout, transport=transport)
async def aclose(self) -> None:
await self._http.aclose()
async def __aenter__(self) -> "AsyncHeaderSpecClient":
return self
async def __aexit__(self, *exc: Any) -> None:
await self.aclose()
async def _send(self, req) -> APIResponse:
method, url, headers, body = req
return _wrap(await self._http.request(method, url, headers=headers, json=body))
async def read(self, schema: str, entity: str, id: Optional[str] = None, options: Optional[HeaderSpecOptions] = None) -> APIResponse:
return await self._send(self._read_req(schema, entity, id, options))
async def create(self, schema: str, entity: str, data: Any, options: Optional[HeaderSpecOptions] = None) -> APIResponse:
return await self._send(self._create_req(schema, entity, data, options))
async def update(self, schema: str, entity: str, id: str, data: Any, options: Optional[HeaderSpecOptions] = None) -> APIResponse:
return await self._send(self._update_req(schema, entity, id, data, options))
async def delete(self, schema: str, entity: str, id: str) -> APIResponse:
return await self._send(self._delete_req(schema, entity, id))
@@ -0,0 +1,76 @@
"""Shared HTTP helpers for the REST clients."""
from __future__ import annotations
from typing import Any, Dict, Mapping, Optional
from urllib.parse import quote
class ResolveSpecError(Exception):
"""Raised on a non-2xx response or an unsuccessful API result."""
def __init__(
self,
message: str,
status_code: Optional[int] = None,
code: Optional[str] = None,
details: Any = None,
detail: Optional[str] = None,
):
super().__init__(message)
self.message = message
self.status_code = status_code
self.code = code
self.details = details
self.detail = detail # server-side reason (funcspec / restheadspec errors)
def merge_headers(*sources: Mapping[str, str]) -> Dict[str, str]:
"""Merge HTTP headers case-insensitively; the last source wins and keeps its spelling."""
result: Dict[str, str] = {}
for source in sources:
for name, value in source.items():
for existing in [k for k in result if k.lower() == name.lower()]:
del result[existing]
result[name] = value
return result
def client_headers(token: Optional[str], headers: Optional[Mapping[str, str]]) -> Dict[str, str]:
"""Content-Type < custom headers < bearer token."""
return merge_headers(
{"Content-Type": "application/json"},
headers or {},
{"Authorization": f"Bearer {token}"} if token else {},
)
def build_url(base_url: str, schema: str, entity: str, id: Optional[Any] = None) -> str:
url = f"{base_url.rstrip('/')}/{quote(schema, safe='')}/{quote(entity, safe='')}"
if id is not None and id != "":
url += f"/{quote(str(id), safe='')}"
return url
def drop_none(d: Mapping[str, Any]) -> Dict[str, Any]:
return {k: v for k, v in d.items() if v is not None}
def parse_json(response: Any) -> Any:
try:
return response.json()
except ValueError:
return None
def error_from(response: Any, data: Any) -> ResolveSpecError:
err = data.get("error") if isinstance(data, dict) else None
err = err if isinstance(err, dict) else {}
text = (response.text or "").strip() if data is None else ""
fallback = text[:200] or f"{response.reason_phrase} ({response.status_code})"
return ResolveSpecError(
err.get("message") or fallback,
status_code=response.status_code,
code=err.get("code"),
details=err.get("details"),
detail=err.get("detail"),
)
@@ -0,0 +1,137 @@
"""ResolveSpec client: JSON body protocol (POST {operation, data, options})."""
from __future__ import annotations
from typing import Any, Dict, List, Mapping, Optional, Tuple
import httpx
from .http import build_url, client_headers, drop_none, error_from, parse_json
from .types import APIResponse, Options, RecordId
def _url_id(id: Optional[RecordId]) -> Optional[str]:
return str(id) if isinstance(id, (int, str)) else None
def _body_id(id: Optional[RecordId]) -> Optional[List[str]]:
return id if isinstance(id, list) else None
class _Base:
def __init__(
self,
base_url: str,
token: Optional[str] = None,
headers: Optional[Mapping[str, str]] = None,
timeout: Optional[float] = 30.0,
):
self.base_url = base_url
self.token = token
self.headers = dict(headers or {}) # snapshot
self.timeout = timeout
def _headers(self) -> Dict[str, str]:
return client_headers(self.token, self.headers)
def _request(
self, method: str, schema: str, entity: str, id: Optional[str], body: Optional[Dict[str, Any]]
) -> Tuple[str, str, Dict[str, str], Optional[Dict[str, Any]]]:
return method, build_url(self.base_url, schema, entity, id), self._headers(), body
@staticmethod
def _result(response: httpx.Response) -> APIResponse:
data = parse_json(response)
if not response.is_success:
raise error_from(response, data)
return data
# request builders (shared by sync and async)
def _metadata_req(self, schema, entity):
return self._request("GET", schema, entity, None, None)
def _read_req(self, schema, entity, id, options):
body = drop_none({"operation": "read", "id": _body_id(id), "options": options})
return self._request("POST", schema, entity, _url_id(id), body)
def _create_req(self, schema, entity, data, options):
body = drop_none({"operation": "create", "data": data, "options": options})
return self._request("POST", schema, entity, None, body)
def _update_req(self, schema, entity, data, id, options):
body = drop_none({"operation": "update", "id": _body_id(id), "data": data, "options": options})
return self._request("POST", schema, entity, _url_id(id), body)
def _delete_req(self, schema, entity, id):
return self._request("POST", schema, entity, str(id), {"operation": "delete"})
class ResolveSpecClient(_Base):
"""Synchronous client. Use as a context manager or call close()."""
def __init__(self, *args: Any, transport: Optional[httpx.BaseTransport] = None, **kwargs: Any):
super().__init__(*args, **kwargs)
self._http = httpx.Client(timeout=self.timeout, transport=transport)
def close(self) -> None:
self._http.close()
def __enter__(self) -> "ResolveSpecClient":
return self
def __exit__(self, *exc: Any) -> None:
self.close()
def _send(self, req) -> APIResponse:
method, url, headers, body = req
return self._result(self._http.request(method, url, headers=headers, json=body))
def get_metadata(self, schema: str, entity: str) -> APIResponse:
return self._send(self._metadata_req(schema, entity))
def read(self, schema: str, entity: str, id: Optional[RecordId] = None, options: Optional[Options] = None) -> APIResponse:
return self._send(self._read_req(schema, entity, id, options))
def create(self, schema: str, entity: str, data: Any, options: Optional[Options] = None) -> APIResponse:
return self._send(self._create_req(schema, entity, data, options))
def update(self, schema: str, entity: str, data: Any, id: Optional[RecordId] = None, options: Optional[Options] = None) -> APIResponse:
return self._send(self._update_req(schema, entity, data, id, options))
def delete(self, schema: str, entity: str, id: Any) -> APIResponse:
return self._send(self._delete_req(schema, entity, id))
class AsyncResolveSpecClient(_Base):
"""Asyncio client. Use as an async context manager or await aclose()."""
def __init__(self, *args: Any, transport: Optional[httpx.AsyncBaseTransport] = None, **kwargs: Any):
super().__init__(*args, **kwargs)
self._http = httpx.AsyncClient(timeout=self.timeout, transport=transport)
async def aclose(self) -> None:
await self._http.aclose()
async def __aenter__(self) -> "AsyncResolveSpecClient":
return self
async def __aexit__(self, *exc: Any) -> None:
await self.aclose()
async def _send(self, req) -> APIResponse:
method, url, headers, body = req
return self._result(await self._http.request(method, url, headers=headers, json=body))
async def get_metadata(self, schema: str, entity: str) -> APIResponse:
return await self._send(self._metadata_req(schema, entity))
async def read(self, schema: str, entity: str, id: Optional[RecordId] = None, options: Optional[Options] = None) -> APIResponse:
return await self._send(self._read_req(schema, entity, id, options))
async def create(self, schema: str, entity: str, data: Any, options: Optional[Options] = None) -> APIResponse:
return await self._send(self._create_req(schema, entity, data, options))
async def update(self, schema: str, entity: str, data: Any, id: Optional[RecordId] = None, options: Optional[Options] = None) -> APIResponse:
return await self._send(self._update_req(schema, entity, data, id, options))
async def delete(self, schema: str, entity: str, id: Any) -> APIResponse:
return await self._send(self._delete_req(schema, entity, id))
@@ -0,0 +1,166 @@
"""Types aligned with Go pkg/common/types.go. Dict keys are the wire names."""
from __future__ import annotations
from typing import Any, Dict, List, NotRequired, TypedDict, Union
Operator = str # eq neq gt gte lt lte like ilike in contains startswith endswith
# between between_inclusive is_null is_not_null
# st_dwithin bbox (spatial) | l2_within cosine_within ip_within (vector)
Operation = str # read | create | update | delete
SortDirection = str # asc | desc | ASC | DESC
VectorMetric = str # l2 | cosine | ip
ResponseFormat = str # simple | detail | syncfusion
RecordId = Union[int, str, List[str]]
class Parameter(TypedDict):
name: str
value: str
sequence: NotRequired[int]
class FilterOption(TypedDict):
column: str
operator: str
value: Any
logic_operator: NotRequired[str] # "AND" | "OR"
class SortOption(TypedDict):
column: str
direction: str
class CustomOperator(TypedDict):
name: str
sql: str
class ComputedColumn(TypedDict):
name: str
expression: str
class PreloadOption(TypedDict, total=False):
relation: str
table_name: str
columns: List[str]
omit_columns: List[str]
sort: List[SortOption]
filters: List[FilterOption]
where: str
limit: int
offset: int
updateable: bool
computed_ql: Dict[str, str]
recursive: bool
primary_key: str
related_key: str
foreign_key: str
recursive_child_key: str
sql_joins: List[str]
join_aliases: List[str]
# `as` is a keyword, so the functional syntax is required.
VectorSearchOption = TypedDict(
"VectorSearchOption",
{
"column": str,
"vector": List[float],
"metric": str, # l2 (default) | cosine | ip
"as": str, # distance column alias, default _distance
"direction": str, # asc (default) | desc
},
total=False,
)
class ExpandOption(TypedDict, total=False):
relation: str
columns: List[str]
class XFiles(TypedDict, total=False):
tablename: str
schema: str
primarykey: str
foreignkey: str
relatedkey: str
sort: List[str]
prefix: str
editable: bool
recursive: bool
expand: bool
rownumber: bool
skipcount: bool
offset: int
limit: int
columns: List[str]
omit_columns: List[str]
cql_columns: List[str]
sql_joins: List[str]
sql_or: List[str]
sql_and: List[str]
parenttables: List["XFiles"]
childtables: List["XFiles"]
filter_fields: List[Dict[str, str]]
cursor_forward: str
cursor_backward: str
class Options(TypedDict, total=False):
preload: List[PreloadOption]
columns: List[str]
omit_columns: List[str]
filters: List[FilterOption]
sort: List[SortOption]
limit: int
offset: int
customOperators: List[CustomOperator]
computedColumns: List[ComputedColumn]
parameters: List[Parameter]
cursor_forward: str
cursor_backward: str
fetch_row_number: str
vector_search: VectorSearchOption
class HeaderSpecOptions(Options, total=False):
"""Options only available to the header-based (restheadspec) protocol."""
expand: List[ExpandOption] # X-Expand
custom_sql_joins: List[str] # X-Custom-SQL-Join
custom_sql_or: List[str] # X-Custom-SQL-Or
search_columns: List[str] # X-SearchCols
advanced_sql: Dict[str, str] # X-AdvSQL-{col}
clean_json: bool # X-Clean-JSON
distinct: bool # X-Distinct
skip_count: bool # X-SkipCount
skip_cache: bool # X-SkipCache
pk_row: str # X-PKRow
response_format: str # X-SimpleApi / X-DetailApi / X-Syncfusion
single_record_as_object: bool # X-Single-Record-As-Object
atomic_transaction: bool # X-Transaction-Atomic
xfiles: XFiles # X-Files
class FuncSpecOptions(TypedDict, total=False):
"""Options understood by funcspec endpoints (sent as X-* headers)."""
filters: List[FilterOption] # eq+AND -> X-FieldFilter; others X-SearchOp / X-SearchOr (one per column)
search_filters: Dict[str, str] # X-SearchFilter-{col}: text ILIKE
custom_sql_where: str # X-Custom-SQL-W
custom_sql_or: str # X-Custom-SQL-Or
sort: List[SortOption] # sent as SQL ORDER BY terms ("col DESC")
limit: int
offset: int
distinct: bool
skip_count: bool
skip_cache: bool
response_format: str # simple | detail | syncfusion
# Responses are plain dicts: {"success", "data", "metadata"?, "error"?}
APIResponse = Dict[str, Any]
@@ -0,0 +1,335 @@
"""WebSocketSpec client (asyncio). Mirrors the Go websocketspec message protocol."""
from __future__ import annotations
import asyncio
import json
import logging
import uuid
from dataclasses import dataclass, field
from typing import Any, Awaitable, Callable, Dict, List, Optional, Union
from websockets.asyncio.client import ClientConnection, connect
from .http import ResolveSpecError
from .types import FilterOption, PreloadOption, SortOption
log = logging.getLogger("resolvespec.websocket")
# Connection states
DISCONNECTED = "disconnected"
CONNECTING = "connecting"
CONNECTED = "connected"
DISCONNECTING = "disconnecting"
RECONNECTING = "reconnecting"
Notification = Dict[str, Any]
Callback = Callable[[Any], Union[None, Awaitable[None]]]
EVENTS = ("connect", "disconnect", "error", "message", "state_change")
@dataclass
class Subscription:
id: str
entity: str
schema: Optional[str] = None
options: Optional[Dict[str, Any]] = None
callback: Optional[Callback] = field(default=None, repr=False)
def _drop_none(d: Dict[str, Any]) -> Dict[str, Any]:
return {k: v for k, v in d.items() if v is not None}
class WebSocketClient:
"""
Usage:
async with WebSocketClient("ws://localhost:8080/ws") as ws:
rows = await ws.read("users", schema="public", limit=10)
Events (`on(event, callback)`): connect, disconnect, error, message, state_change.
Callbacks may be sync or async.
"""
def __init__(
self,
url: str,
*,
reconnect: bool = True,
reconnect_interval: float = 3.0,
max_reconnect_attempts: int = 10,
heartbeat_interval: float = 30.0,
request_timeout: float = 30.0,
subscribe_timeout: float = 10.0,
headers: Optional[Dict[str, str]] = None,
):
self.url = url
self.reconnect = reconnect
self.reconnect_interval = reconnect_interval
self.max_reconnect_attempts = max_reconnect_attempts
self.heartbeat_interval = heartbeat_interval
self.request_timeout = request_timeout
self.subscribe_timeout = subscribe_timeout
self.headers = dict(headers or {})
self._ws: Optional[ClientConnection] = None
self._state = DISCONNECTED
self._pending: Dict[str, "asyncio.Future[Dict[str, Any]]"] = {}
self._subscriptions: Dict[str, Subscription] = {}
self._listeners: Dict[str, Callback] = {}
self._tasks: List["asyncio.Task[Any]"] = []
self._reader: Optional["asyncio.Task[Any]"] = None
self._manual_close = False
# ---- lifecycle -------------------------------------------------------
async def __aenter__(self) -> "WebSocketClient":
await self.connect()
return self
async def __aexit__(self, *exc: Any) -> None:
await self.close()
async def connect(self) -> None:
if self.is_connected():
return
self._manual_close = False
self._set_state(CONNECTING)
try:
self._ws = await connect(self.url, additional_headers=self.headers or None)
except Exception as e:
self._set_state(DISCONNECTED)
await self._emit("error", e)
raise
self._set_state(CONNECTED)
self._reader = asyncio.create_task(self._read_loop(self._ws))
self._heartbeat = asyncio.create_task(self._heartbeat_loop())
await self._emit("connect")
async def close(self) -> None:
self._manual_close = True
self._set_state(DISCONNECTING)
for t in (self._reader, getattr(self, "_heartbeat", None), getattr(self, "_reconnect_task", None)):
if t and t is not asyncio.current_task():
t.cancel()
if self._ws:
await self._ws.close()
self._ws = None
self._fail_pending(ResolveSpecError("WebSocket closed"))
self._set_state(DISCONNECTED)
def is_connected(self) -> bool:
return self._ws is not None and self._state == CONNECTED
@property
def state(self) -> str:
return self._state
def on(self, event: str, callback: Callback) -> None:
if event not in EVENTS:
raise ValueError(f"unknown event {event!r}; expected one of {EVENTS}")
self._listeners[event] = callback
def off(self, event: str) -> None:
self._listeners.pop(event, None)
def get_subscriptions(self) -> List[Subscription]:
return list(self._subscriptions.values())
# ---- operations ------------------------------------------------------
async def request(
self,
operation: str,
entity: str,
*,
schema: Optional[str] = None,
record_id: Optional[str] = None,
data: Any = None,
options: Optional[Dict[str, Any]] = None,
) -> Any:
message = _drop_none({
"type": "request",
"operation": operation,
"entity": entity,
"schema": schema,
"record_id": record_id,
"data": data,
"options": options,
})
response = await self._call(message, self.request_timeout, "Request")
return response.get("data")
async def read(
self,
entity: str,
*,
schema: Optional[str] = None,
record_id: Optional[str] = None,
filters: Optional[List[FilterOption]] = None,
columns: Optional[List[str]] = None,
sort: Optional[List[SortOption]] = None,
preload: Optional[List[PreloadOption]] = None,
limit: Optional[int] = None,
offset: Optional[int] = None,
) -> Any:
options = _drop_none({
"filters": filters, "columns": columns, "sort": sort,
"preload": preload, "limit": limit, "offset": offset,
})
return await self.request("read", entity, schema=schema, record_id=record_id, options=options)
async def create(self, entity: str, data: Any, *, schema: Optional[str] = None) -> Any:
return await self.request("create", entity, schema=schema, data=data)
async def update(self, entity: str, id: str, data: Any, *, schema: Optional[str] = None) -> Any:
return await self.request("update", entity, schema=schema, record_id=id, data=data)
async def delete(self, entity: str, id: str, *, schema: Optional[str] = None) -> None:
await self.request("delete", entity, schema=schema, record_id=id)
async def meta(self, entity: str, *, schema: Optional[str] = None) -> Any:
return await self.request("meta", entity, schema=schema)
async def subscribe(
self,
entity: str,
callback: Callback,
*,
schema: Optional[str] = None,
filters: Optional[List[FilterOption]] = None,
) -> str:
message = _drop_none({
"type": "subscription",
"operation": "subscribe",
"entity": entity,
"schema": schema,
"options": _drop_none({"filters": filters}),
})
response = await self._call(message, self.subscribe_timeout, "Subscription")
sub_id = (response.get("data") or {}).get("subscription_id")
if not sub_id:
raise ResolveSpecError("Subscription failed")
self._subscriptions[sub_id] = Subscription(
sub_id, entity, schema, _drop_none({"filters": filters}) or None, callback
)
return sub_id
async def unsubscribe(self, subscription_id: str) -> None:
message = {"type": "subscription", "operation": "unsubscribe", "subscription_id": subscription_id}
await self._call(message, self.subscribe_timeout, "Unsubscribe")
self._subscriptions.pop(subscription_id, None)
# ---- internals -------------------------------------------------------
async def _call(self, message: Dict[str, Any], timeout: float, what: str) -> Dict[str, Any]:
self._ensure_connected()
mid = str(uuid.uuid4())
message["id"] = mid
fut: "asyncio.Future[Dict[str, Any]]" = asyncio.get_running_loop().create_future()
self._pending[mid] = fut
try:
await self._ws.send(json.dumps(message)) # type: ignore[union-attr]
response = await asyncio.wait_for(fut, timeout)
except asyncio.TimeoutError:
raise ResolveSpecError(f"{what} timeout") from None
finally:
self._pending.pop(mid, None)
if not response.get("success"):
err = response.get("error") or {}
raise ResolveSpecError(
err.get("message") or f"{what} failed", code=err.get("code"), details=err.get("details")
)
return response
def _ensure_connected(self) -> None:
if not self.is_connected():
raise ResolveSpecError("WebSocket is not connected. Call connect() first.")
def _fail_pending(self, exc: Exception) -> None:
for fut in self._pending.values():
if not fut.done():
fut.set_exception(exc)
self._pending.clear()
async def _read_loop(self, ws: ClientConnection) -> None:
try:
async for raw in ws:
await self._handle_message(raw)
except asyncio.CancelledError:
raise
except Exception as e: # connection error
await self._emit("error", e)
# connection ended
if ws is not self._ws:
return
self._ws = None
if hb := getattr(self, "_heartbeat", None):
hb.cancel()
self._fail_pending(ResolveSpecError("WebSocket disconnected"))
self._set_state(DISCONNECTED)
await self._emit("disconnect", ws.close_code, ws.close_reason)
if self.reconnect and not self._manual_close:
self._reconnect_task = asyncio.create_task(self._reconnect())
async def _reconnect(self) -> None:
for attempt in range(1, self.max_reconnect_attempts + 1):
if self._manual_close:
return
log.debug("Reconnection attempt %d/%d", attempt, self.max_reconnect_attempts)
self._set_state(RECONNECTING)
await asyncio.sleep(self.reconnect_interval)
try:
await self.connect()
return
except Exception as e:
log.debug("Reconnection failed: %s", e)
self._set_state(DISCONNECTED)
async def _handle_message(self, raw: Union[str, bytes]) -> None:
try:
message = json.loads(raw)
except ValueError as e:
log.debug("Error parsing message: %s", e)
return
await self._emit("message", message)
kind = message.get("type")
if kind == "response":
fut = self._pending.get(message.get("id"))
if fut and not fut.done():
fut.set_result(message)
elif kind == "notification":
sub = self._subscriptions.get(message.get("subscription_id"))
if sub and sub.callback:
await _maybe_await(sub.callback(message))
elif kind != "pong":
log.debug("Unknown message type: %s", kind)
async def _heartbeat_loop(self) -> None:
try:
while True:
await asyncio.sleep(self.heartbeat_interval)
if self.is_connected():
await self._ws.send(json.dumps({"id": str(uuid.uuid4()), "type": "ping"})) # type: ignore[union-attr]
except asyncio.CancelledError:
raise
except Exception as e:
log.debug("Heartbeat failed: %s", e)
def _set_state(self, state: str) -> None:
if self._state != state:
self._state = state
cb = self._listeners.get("state_change")
if cb:
res = cb(state)
if asyncio.iscoroutine(res):
asyncio.ensure_future(res)
async def _emit(self, event: str, *args: Any) -> None:
cb = self._listeners.get(event)
if cb:
await _maybe_await(cb(*args))
async def _maybe_await(result: Any) -> None:
if asyncio.iscoroutine(result) or isinstance(result, asyncio.Future):
await result
@@ -0,0 +1,132 @@
import httpx
import pytest
from resolvespec import AsyncFuncSpecClient, FuncSpecClient, ResolveSpecError
from resolvespec.funcspec import build_headers, build_query
from resolvespec.headerspec import decode_header_value
def make(handler, **kw):
return FuncSpecClient("http://localhost:3000", "tok", transport=httpx.MockTransport(handler), **kw)
def capture(status=200, body=None, headers=None):
seen = []
def handler(req):
seen.append(req)
return httpx.Response(status, json=body if body is not None else [], headers=headers)
return seen, handler
def test_filters():
h = build_headers({"filters": [
{"column": "status", "operator": "eq", "value": "active"},
{"column": "age", "operator": "gte", "value": 18},
{"column": "name", "operator": "contains", "value": "x", "logic_operator": "OR"},
{"column": "deleted", "operator": "is_null", "value": None},
{"column": "id", "operator": "in", "value": [1, 2]},
{"column": "p", "operator": "between_inclusive", "value": [1, 5]},
]})
assert h == {
"X-FieldFilter-status": "active",
"X-SearchOp-greaterthanorequal-age": "18",
"X-SearchOr-contains-name": "x",
"X-SearchOp-empty-deleted": "",
"X-SearchOp-in-id": "1,2",
"X-SearchOp-betweeninclusive-p": "1,5",
}
def test_sort_is_sql_not_prefixed():
# server inserts sort verbatim into ORDER BY; "-col" would negate the column
h = build_headers({"sort": [{"column": "name", "direction": "asc"}, {"column": "created_at", "direction": "DESC"}]})
assert h["X-Sort"] == "name ASC,created_at DESC"
def test_misc_options():
h = build_headers({
"search_filters": {"name": "bob"}, "custom_sql_where": "a = 1", "custom_sql_or": "b = 2",
"limit": 5, "offset": 10, "distinct": True, "skip_count": True, "skip_cache": False,
"response_format": "syncfusion",
})
assert h == {
"X-SearchFilter-name": "bob", "X-Custom-SQL-W": "a = 1", "X-Custom-SQL-Or": "b = 2",
"X-Limit": "5", "X-Offset": "10", "X-Distinct": "true", "X-SkipCount": "true",
"X-SkipCache": "false", "X-Syncfusion": "true",
}
def test_ambiguous_values_are_encoded():
h = build_headers({"custom_sql_where": "name = 'café'", "filters": [{"column": "c", "operator": "eq", "value": " pad "}]})
assert h["X-Custom-SQL-W"].startswith("ZIP_")
assert decode_header_value(h["X-Custom-SQL-W"]) == "name = 'café'"
assert decode_header_value(h["X-FieldFilter-c"]) == " pad "
def test_build_query():
q = build_query({"p-id": 5, "flag": True, "ids": [1, 2], "skip": None, "m": "match=ab"})
assert q == {"p-id": "5", "flag": "true", "ids": ["1", "2"], "m": "match=ab"}
def test_query_list_request_and_metadata():
seen, h = capture(206, [{"id": 1}, {"id": 2}], {"content-range": "items 10-12/50"})
with make(h) as c:
res = c.query_list("/api/orders", {"p-status": "open", "id": [1, 2]}, {"limit": 2, "offset": 10})
r = seen[0]
assert r.method == "GET"
assert r.url.path == "/api/orders"
assert r.url.params.multi_items() == [("p-status", "open"), ("id", "1"), ("id", "2")]
assert r.headers["x-limit"] == "2" and r.headers["authorization"] == "Bearer tok"
assert res == {
"success": True,
"data": [{"id": 1}, {"id": 2}],
"metadata": {"total": 50, "count": 2, "filtered": 50, "offset": 10, "limit": 2},
}
def test_query_list_empty_result():
seen, h = capture(200, [], {"content-range": "items 0-0/0"})
with make(h) as c:
assert c.query_list("orders")["metadata"]["total"] == 0
assert seen[0].url.path == "/orders"
def test_query_single_has_no_metadata_and_method():
seen, h = capture(200, {"id": 1})
with make(h) as c:
res = c.query("api/order", method="post")
assert seen[0].method == "POST"
assert res == {"success": True, "data": {"id": 1}}
def test_detail_format_data_passthrough():
body = {"items": [{"a": 1}], "count": "1", "total": "1", "tablename": "/x", "tableprefix": "gsql"}
_, h = capture(200, body, {"content-range": "items 0-1/1"})
with make(h) as c:
assert c.query_list("x", options={"response_format": "detail"})["data"] == body
def test_server_error_shape():
err = {"success": False, "error": {"code": "query_failed", "message": "Failed to retrieve records", "detail": "no such column", "sql": "SELECT"}}
_, h = capture(400, err)
with make(h) as c:
with pytest.raises(ResolveSpecError, match="Failed to retrieve") as ei:
c.query_list("x")
assert ei.value.code == "query_failed" and ei.value.detail == "no such column" and ei.value.status_code == 400
def test_plain_text_panic_error():
with make(lambda r: httpx.Response(500, text="Internal server error: boom")) as c:
with pytest.raises(ResolveSpecError, match="boom"):
c.query("x")
async def test_async():
async def handler(req):
return httpx.Response(200, json=[{"id": 1}], headers={"content-range": "items 0-1/1"})
async with AsyncFuncSpecClient("http://localhost:3000", transport=httpx.MockTransport(handler)) as c:
assert (await c.query_list("x"))["metadata"]["total"] == 1
assert (await c.query("x"))["data"] == [{"id": 1}]
@@ -0,0 +1,236 @@
import json
import httpx
import pytest
from resolvespec import (
AsyncHeaderSpecClient,
HeaderSpecClient,
ResolveSpecError,
build_headers,
decode_header_value,
encode_header_value,
get_headerspec_client,
)
import base64
CFG = dict(base_url="http://localhost:3000", token="tok")
# ---- build_headers (ported from headerspec.test.ts) ----
def test_preload_shared_where():
h = build_headers({"preload": [
{"relation": "Items", "columns": ["id"], "where": "active = true"},
{"relation": "Tags", "where": "active = true"},
]})
assert h["X-Preload"] == "Items:id|Tags"
assert h["X-Preload-Where"] == "active = true"
def test_preload_mixed_where_numbered():
h = build_headers({"preload": [
{"relation": "Items", "where": "a = 1"},
{"relation": "Category"},
{"relation": "Tags", "where": "b = 2"},
]})
assert h["X-Preload"] == "Category"
assert "X-Preload-Where" not in h
assert h["X-Preload-1"] == "Items" and h["X-Preload-1-Where"] == "a = 1"
assert h["X-Preload-2"] == "Tags" and h["X-Preload-2-Where"] == "b = 2"
def test_expand_joins_or_searchcols_advsql():
h = build_headers({
"expand": [{"relation": "Dept", "columns": ["id", "name"]}, {"relation": "Role"}],
"custom_sql_joins": ["LEFT JOIN a ON a.id = b.id", "INNER JOIN c ON c.id = b.cid"],
"custom_sql_or": ["x = 1", "y = 2"],
"search_columns": ["name", "email"],
"advanced_sql": {"total": "a + b"},
})
assert h["X-Expand"] == "Dept:id,name|Role"
assert h["X-Custom-SQL-Join"] == "LEFT JOIN a ON a.id = b.id|INNER JOIN c ON c.id = b.cid"
assert h["X-Custom-SQL-Or"] == "x = 1 OR y = 2"
assert h["X-SearchCols"] == "name,email"
assert h["X-AdvSQL-total"] == "a + b"
def test_flags_pkrow_format():
h = build_headers({
"clean_json": True, "distinct": True, "skip_count": True, "skip_cache": False,
"atomic_transaction": True, "single_record_as_object": False,
"pk_row": "42", "response_format": "detail",
})
assert h["X-Clean-JSON"] == "true"
assert h["X-Distinct"] == "true"
assert h["X-SkipCount"] == "true"
assert h["X-SkipCache"] == "false"
assert h["X-Transaction-Atomic"] == "true"
assert h["X-Single-Record-As-Object"] == "false"
assert h["X-PKRow"] == "42"
assert h["X-DetailApi"] == "true"
def test_spatial_and_vector_filters():
h = build_headers({"filters": [
{"column": "geom", "operator": "st_dwithin", "value": {"geom": "POINT(0 0)", "distance": 5}, "logic_operator": "OR"},
{"column": "emb", "operator": "cosine_within", "value": {"vector": [1, 2], "distance": 0.3}},
]})
assert json.loads(h["X-SpatialFilter-geom"]) == {
"op": "st_dwithin", "value": {"geom": "POINT(0 0)", "distance": 5}, "logic": "or"}
assert json.loads(h["X-VectorFilter-emb"])["op"] == "cosine_within"
def test_vector_search():
h = build_headers({"vector_search": {"column": "emb", "vector": [0.1, 0.2], "metric": "cosine", "as": "dist", "direction": "desc"}})
assert h["X-Vector-Search-emb"] == "cosine"
assert h["X-Vector-Search-Vector"] == "[0.1,0.2]"
assert h["X-Vector-Search-As"] == "dist"
assert h["X-Vector-Search-Dir"] == "desc"
def test_xfiles_zip():
xf = {"tablename": "users", "prefix": "USR", "limit": 10}
h = build_headers({"xfiles": xf})
assert h["X-Files"].startswith("ZIP_")
assert json.loads(decode_header_value(h["X-Files"])) == xf
def test_columns_and_omit():
assert build_headers({"columns": ["id", "name", "email"]})["X-Select-Fields"] == "id,name,email"
assert build_headers({"omit_columns": ["secret", "internal"]})["X-Not-Select-Fields"] == "secret,internal"
def test_filters():
assert build_headers({"filters": [{"column": "status", "operator": "eq", "value": "active"}]})["X-FieldFilter-status"] == "active"
assert build_headers({"filters": [{"column": "age", "operator": "gte", "value": 18}]})["X-SearchOp-greaterthanorequal-age"] == "18"
assert build_headers({"filters": [{"column": "name", "operator": "contains", "value": "test", "logic_operator": "OR"}]})["X-SearchOr-contains-name"] == "test"
assert build_headers({"filters": [{"column": "price", "operator": "between", "value": [10, 100]}]})["X-SearchOp-between-price"] == "10,100"
assert build_headers({"filters": [{"column": "deleted_at", "operator": "is_null", "value": None}]})["X-SearchOp-empty-deleted_at"] == ""
assert build_headers({"filters": [{"column": "id", "operator": "in", "value": [1, 2, 3]}]})["X-SearchOp-in-id"] == "1,2,3"
assert build_headers({"filters": [{"column": "a", "operator": "eq", "value": True}]})["X-FieldFilter-a"] == "true"
def test_sort_pagination_cursor():
h = build_headers({
"sort": [{"column": "name", "direction": "asc"}, {"column": "created_at", "direction": "DESC"}],
"limit": 25, "offset": 0, "cursor_forward": "abc", "cursor_backward": "xyz",
})
assert h["X-Sort"] == "+name,-created_at"
assert h["X-Limit"] == "25" and h["X-Offset"] == "0"
assert h["X-Cursor-Forward"] == "abc" and h["X-Cursor-Backward"] == "xyz"
def test_preload_basic_rownumber_computed_custom():
h = build_headers({
"preload": [{"relation": "Items", "columns": ["id", "name"]}, {"relation": "Category"}],
"fetch_row_number": "42",
"computedColumns": [{"name": "total", "expression": "price * qty"}],
"customOperators": [{"name": "a", "sql": "status = 'active'"}, {"name": "v", "sql": "verified = true"}],
})
assert h["X-Preload"] == "Items:id,name|Category"
assert h["X-Fetch-RowNumber"] == "42"
assert h["X-CQL-SEL-total"] == "price * qty"
assert h["X-Custom-SQL-W"] == "status = 'active' AND verified = true"
def test_empty_options():
assert build_headers({}) == {}
# ---- encode / decode ----
def test_roundtrip():
for s in ("some complex value with spaces & symbols!", "café ☕ 你好"):
enc = encode_header_value(s)
assert enc.startswith("ZIP_")
assert decode_header_value(enc) == s
def test_decode_double_underscore_and_plain():
assert decode_header_value("__" + base64.b64encode(b"hello").decode()) == "hello"
assert decode_header_value("__" + base64.b64encode("café ☕".encode()).decode()) == "café ☕"
assert decode_header_value("plain") == "plain"
def test_decode_nested():
assert decode_header_value(encode_header_value(encode_header_value("x"))) == "x"
# ---- client ----
def make(handler, cls=HeaderSpecClient, **kw):
return cls(**{**CFG, **kw}, transport=httpx.MockTransport(handler))
def test_read_sends_get_with_headers():
seen = []
def handler(req):
seen.append(req)
return httpx.Response(200, json=[{"id": 1}], headers={"content-range": "0-9/100", "x-limit": "10"})
with make(handler) as c:
res = c.read("public", "users", options={"columns": ["id", "name"], "limit": 10})
r = seen[0]
assert str(r.url) == "http://localhost:3000/public/users"
assert r.method == "GET"
assert r.headers["x-select-fields"] == "id,name"
assert r.headers["x-limit"] == "10"
assert r.headers["authorization"] == "Bearer tok"
assert res["success"] is True
assert res["data"] == [{"id": 1}]
assert res["metadata"] == {"count": 100, "total": 100, "filtered": 100, "offset": 0, "limit": 10}
def test_metadata_defaults_without_content_range():
with make(lambda r: httpx.Response(200, json=[])) as c:
assert c.read("public", "users")["metadata"]["total"] == 0
def test_read_with_id_create_update_delete():
seen = []
def handler(req):
seen.append(req)
return httpx.Response(200, json={})
with make(handler) as c:
c.read("public", "users", "42")
c.create("public", "users", {"name": "Test"})
c.update("public", "users", "1", {"name": "Updated"}, {"filters": [{"column": "active", "operator": "eq", "value": True}]})
c.delete("public", "users", "1")
assert str(seen[0].url) == "http://localhost:3000/public/users/42"
assert seen[1].method == "POST" and json.loads(seen[1].content) == {"name": "Test"}
assert seen[2].method == "PUT" and str(seen[2].url).endswith("/public/users/1")
assert seen[2].headers["x-fieldfilter-active"] == "true"
assert seen[3].method == "DELETE"
def test_error_response():
with make(lambda r: httpx.Response(400, json={"error": {"code": "err", "message": "fail"}})) as c:
with pytest.raises(ResolveSpecError, match="fail") as ei:
c.read("public", "users")
assert ei.value.status_code == 400 and ei.value.code == "err"
def test_error_non_json():
with make(lambda r: httpx.Response(502, text="bad gateway")) as c:
with pytest.raises(ResolveSpecError, match="bad gateway") as ei:
c.read("public", "users")
assert ei.value.status_code == 502
async def test_async_client():
async def handler(req):
return httpx.Response(200, json=[{"id": 1}])
async with AsyncHeaderSpecClient(**CFG, transport=httpx.MockTransport(handler)) as c:
res = await c.read("public", "users", options={"limit": 1})
assert res["data"] == [{"id": 1}]
def test_singleton():
a = get_headerspec_client("http://hs-singleton:3000")
assert a is get_headerspec_client("http://hs-singleton:3000")
assert a is not get_headerspec_client("http://hs-singleton-b:3000")
@@ -0,0 +1,210 @@
import json
import httpx
import pytest
from resolvespec import (
AsyncHeaderSpecClient,
AsyncResolveSpecClient,
HeaderSpecClient,
ResolveSpecClient,
ResolveSpecError,
get_headerspec_client,
get_resolvespec_client,
)
CFG = dict(base_url="http://localhost:3000", token="test-token")
def make(handler, **kw):
return ResolveSpecClient(**{**CFG, **kw}, transport=httpx.MockTransport(handler))
def ok(_req):
return httpx.Response(200, json={"success": True, "data": [{"id": 1}]})
def capture():
seen = []
def handler(req):
seen.append(req)
return httpx.Response(200, json={"success": True, "data": {"id": 1, "name": "Test"}})
return seen, handler
def body(req):
return json.loads(req.content)
def test_read_with_numeric_id():
seen, h = capture()
with make(h) as c:
assert c.read("public", "users", 1)["success"] is True
r = seen[0]
assert str(r.url) == "http://localhost:3000/public/users/1"
assert r.method == "POST"
assert r.headers["authorization"] == "Bearer test-token"
assert r.headers["content-type"] == "application/json"
assert body(r) == {"operation": "read"}
def test_read_array_id_goes_in_body():
seen, h = capture()
with make(h) as c:
c.read("public", "users", ["1", "2"])
assert str(seen[0].url) == "http://localhost:3000/public/users"
assert body(seen[0])["id"] == ["1", "2"]
def test_read_options_passthrough():
seen, h = capture()
opts = {
"columns": ["id", "name"], "omit_columns": ["secret"],
"filters": [{"column": "active", "operator": "eq", "value": True}],
"sort": [{"column": "name", "direction": "asc"}],
"limit": 10, "offset": 0, "cursor_forward": "cursor1", "fetch_row_number": "5",
"customOperators": [{"name": "x", "sql": "a = 1"}],
}
with make(h) as c:
c.read("public", "users", options=opts)
assert body(seen[0])["options"] == opts
def test_create():
seen, h = capture()
with make(h) as c:
res = c.create("public", "users", {"name": "Test"})
assert res["data"]["name"] == "Test"
assert body(seen[0]) == {"operation": "create", "data": {"name": "Test"}}
def test_create_batch():
seen, h = capture()
with make(h) as c:
c.create("public", "users", [{"a": 1}, {"a": 2}])
assert body(seen[0])["data"] == [{"a": 1}, {"a": 2}]
def test_update_with_id_in_url_and_array():
seen, h = capture()
with make(h) as c:
c.update("public", "users", {"name": "X"}, 5)
c.update("public", "users", {"name": "X"}, ["1", "2"])
assert str(seen[0].url).endswith("/public/users/5")
assert body(seen[0]) == {"operation": "update", "data": {"name": "X"}}
assert str(seen[1].url).endswith("/public/users")
assert body(seen[1])["id"] == ["1", "2"]
def test_update_preserves_empty_string_and_null():
seen, h = capture()
with make(h) as c:
c.update("public", "users", {"a": "", "b": None}, 1)
assert body(seen[0])["data"] == {"a": "", "b": None}
def test_delete():
seen, h = capture()
with make(h) as c:
c.delete("public", "users", 1)
assert str(seen[0].url).endswith("/public/users/1")
assert body(seen[0]) == {"operation": "delete"}
def test_get_metadata():
seen, h = capture()
with make(h) as c:
c.get_metadata("public", "users")
assert seen[0].method == "GET"
assert str(seen[0].url) == "http://localhost:3000/public/users"
assert not seen[0].content
def test_error_uses_server_message():
with make(lambda r: httpx.Response(404, json={"success": False, "error": {"code": "not_found", "message": "nope"}})) as c:
with pytest.raises(ResolveSpecError, match="nope") as ei:
c.read("public", "users", 1)
assert ei.value.status_code == 404 and ei.value.code == "not_found"
def test_id_is_url_quoted():
seen, h = capture()
with make(h) as c:
c.read("public", "users", "a/b")
assert str(seen[0].url).endswith("/public/users/a%2Fb")
def test_trailing_slash_base_url():
seen, h = capture()
with make(h, base_url="http://localhost:3000/") as c:
c.read("public", "users")
assert str(seen[0].url) == "http://localhost:3000/public/users"
async def test_async_client():
async def handler(req):
return httpx.Response(200, json={"success": True, "data": [1]})
async with AsyncResolveSpecClient(**CFG, transport=httpx.MockTransport(handler)) as c:
assert (await c.read("public", "users"))["data"] == [1]
assert (await c.create("public", "users", {}))["success"]
assert (await c.update("public", "users", {}, 1))["success"]
assert (await c.delete("public", "users", 1))["success"]
assert (await c.get_metadata("public", "users"))["success"]
# ---- custom headers (ported from custom-headers.test.ts) ----
@pytest.mark.parametrize("cls", [ResolveSpecClient, HeaderSpecClient])
def test_custom_headers_on_every_op_case_insensitive(cls):
seen = []
def handler(req):
seen.append(req)
return httpx.Response(200, json={"success": True, "data": []})
headers = {"X-Tenant": "acme", "authorization": "Basic ignored",
"content-type": "application/custom+json", "x-limit": "99"}
with cls("http://localhost:3000", "tok", headers, transport=httpx.MockTransport(handler)) as c:
c.read("public", "users", options={"limit": 10})
c.create("public", "users", {})
if cls is ResolveSpecClient:
c.update("public", "users", {}, "1")
c.get_metadata("public", "users")
else:
c.update("public", "users", "1", {})
c.delete("public", "users", "1")
for r in seen:
assert r.headers["x-tenant"] == "acme"
assert r.headers["authorization"] == "Bearer tok"
assert r.headers["content-type"] == "application/custom+json"
if cls is HeaderSpecClient:
assert seen[0].headers["x-limit"] == "10"
assert headers["authorization"] == "Basic ignored"
assert headers["x-limit"] == "99"
@pytest.mark.parametrize("cls", [ResolveSpecClient, HeaderSpecClient])
def test_custom_auth_without_token(cls):
seen = []
def handler(req):
seen.append(req)
return httpx.Response(200, json={"success": True, "data": []})
with cls("http://localhost:3000", headers={"Authorization": "Basic custom"}, transport=httpx.MockTransport(handler)) as c:
c.read("public", "users")
assert seen[0].headers["authorization"] == "Basic custom"
@pytest.mark.parametrize("factory", [get_resolvespec_client, get_headerspec_client])
def test_cache_isolation_and_snapshot(factory):
headers = {"X-Tenant": "acme", "X-App": "grid"}
first = factory("http://tenant-cache", "one", headers)
assert factory("http://tenant-cache", "one", {"x-app": "grid", "x-tenant": "acme"}) is first
assert factory("http://tenant-cache", "two", headers) is not first
headers["X-Tenant"] = "other"
assert factory("http://tenant-cache", "one", headers) is not first
assert first.headers["X-Tenant"] == "acme"
@@ -0,0 +1,152 @@
import asyncio
import json
import pytest
from websockets.asyncio.server import serve
from resolvespec import ResolveSpecError, WebSocketClient
class Server:
"""Minimal in-process WebSocketSpec server."""
def __init__(self):
self.received = []
self.conns = set()
self.respond = True
async def handler(self, ws):
self.conns.add(ws)
try:
async for raw in ws:
msg = json.loads(raw)
self.received.append(msg)
if msg["type"] == "ping":
await ws.send(json.dumps({"type": "pong"}))
continue
if not self.respond:
continue
await ws.send(json.dumps(self.reply(msg)))
finally:
self.conns.discard(ws)
def reply(self, msg):
base = {"id": msg["id"], "type": "response", "success": True, "timestamp": "t"}
if msg["type"] == "subscription" and msg["operation"] == "subscribe":
return {**base, "data": {"subscription_id": "sub-1"}}
if msg.get("entity") == "fail":
return {**base, "success": False, "error": {"code": "bad", "message": "boom"}}
return {**base, "data": {"echo": msg.get("operation"), "record_id": msg.get("record_id")}}
@pytest.fixture
async def server():
s = Server()
async with serve(s.handler, "127.0.0.1", 0) as srv:
s.url = "ws://127.0.0.1:%d" % srv.sockets[0].getsockname()[1]
yield s
async def test_operations_and_message_shape(server):
async with WebSocketClient(server.url, reconnect=False) as c:
assert c.state == "connected"
assert await c.read("users", schema="public", record_id="1", limit=5, filters=[{"column": "a", "operator": "eq", "value": 1}]) == {"echo": "read", "record_id": "1"}
await c.create("users", {"n": 1}, schema="public")
await c.update("users", "2", {"n": 2})
await c.delete("users", "3")
await c.meta("users")
m = server.received
assert m[0]["type"] == "request" and m[0]["operation"] == "read"
assert m[0]["schema"] == "public" and m[0]["record_id"] == "1"
assert m[0]["options"] == {"filters": [{"column": "a", "operator": "eq", "value": 1}], "limit": 5}
assert m[1]["data"] == {"n": 1}
assert m[2]["record_id"] == "2"
assert [x["operation"] for x in m] == ["read", "create", "update", "delete", "meta"]
assert "schema" not in m[2]
assert len({x["id"] for x in m}) == 5
async def test_error_response_raises(server):
async with WebSocketClient(server.url, reconnect=False) as c:
with pytest.raises(ResolveSpecError, match="boom") as ei:
await c.read("fail")
assert ei.value.code == "bad"
async def test_request_timeout(server):
server.respond = False
async with WebSocketClient(server.url, reconnect=False, request_timeout=0.1) as c:
with pytest.raises(ResolveSpecError, match="timeout"):
await c.read("users")
assert not c._pending
async def test_not_connected_raises():
c = WebSocketClient("ws://127.0.0.1:1")
with pytest.raises(ResolveSpecError, match="not connected"):
await c.read("users")
async def test_subscribe_notify_unsubscribe(server):
got = asyncio.Queue()
async with WebSocketClient(server.url, reconnect=False) as c:
sid = await c.subscribe("users", got.put, schema="public", filters=[{"column": "a", "operator": "eq", "value": 1}])
assert sid == "sub-1"
assert [s.id for s in c.get_subscriptions()] == ["sub-1"]
assert server.received[0]["operation"] == "subscribe"
assert server.received[0]["options"] == {"filters": [{"column": "a", "operator": "eq", "value": 1}]}
for ws in server.conns:
await ws.send(json.dumps({"type": "notification", "operation": "create", "subscription_id": "sub-1",
"entity": "users", "data": {"id": 9}, "timestamp": "t"}))
n = await asyncio.wait_for(got.get(), 2)
assert n["data"] == {"id": 9}
await c.unsubscribe("sub-1")
assert c.get_subscriptions() == []
assert server.received[-1] == {**server.received[-1], "operation": "unsubscribe", "subscription_id": "sub-1"}
async def test_events_and_heartbeat(server):
events = []
c = WebSocketClient(server.url, reconnect=False, heartbeat_interval=0.05)
c.on("connect", lambda: events.append("connect"))
c.on("state_change", lambda s: events.append(s))
c.on("message", lambda m: events.append(("msg", m["type"])))
await c.connect()
await asyncio.sleep(0.2)
await c.close()
assert events[:3] == ["connecting", "connected", "connect"]
assert ("msg", "pong") in events
assert events[-1] == "disconnected"
assert any(m["type"] == "ping" for m in server.received)
with pytest.raises(ValueError):
c.on("bogus", lambda: None)
async def test_reconnect_after_server_drop(server):
states = []
c = WebSocketClient(server.url, reconnect=True, reconnect_interval=0.05)
c.on("state_change", states.append)
await c.connect()
for ws in list(server.conns):
await ws.close()
for _ in range(100):
if states.count("connected") >= 2:
break
await asyncio.sleep(0.05)
assert "reconnecting" in states
assert c.is_connected()
assert (await c.read("users"))["echo"] == "read"
await c.close()
async def test_pending_requests_fail_on_disconnect(server):
server.respond = False
c = WebSocketClient(server.url, reconnect=False)
await c.connect()
task = asyncio.create_task(c.read("users"))
await asyncio.sleep(0.05)
for ws in list(server.conns):
await ws.close()
with pytest.raises(ResolveSpecError, match="disconnected"):
await asyncio.wait_for(task, 2)
await c.close()
@@ -4,34 +4,34 @@
### 1. ResolveSpec Client API
- [ ] Core API implementation (read, create, update, delete, get_metadata)
- [ ] Unit tests for API functions
- [x] Core API implementation (read, create, update, delete, get_metadata)
- [x] Unit tests for API functions
- [ ] Integration tests with server
- [ ] Error handling and edge cases
- [x] Error handling and edge cases
### 2. HeaderSpec Client API
- [ ] Client API implementation
- [ ] Unit tests
- [x] Client API implementation
- [x] Unit tests
- [ ] Integration tests with server
### 3. FunctionSpec Client API
- [ ] Client API implementation
- [ ] Unit tests
- [x] Client API implementation
- [x] Unit tests
- [ ] Integration tests with server
### 4. WebSocketSpec Client API
- [ ] WebSocketClient class implementation (read, create, update, delete, meta, subscribe, unsubscribe)
- [ ] Unit tests for WebSocketClient
- [ ] Connection handling tests
- [ ] Subscription tests
- [x] WebSocketClient class implementation (read, create, update, delete, meta, subscribe, unsubscribe)
- [x] Unit tests for WebSocketClient
- [x] Connection handling tests
- [x] Subscription tests
- [ ] Integration tests with server
### 5. Testing Infrastructure
- [ ] Set up test framework (pytest)
- [x] Set up test framework (pytest)
- [ ] Configure test coverage reporting (pytest-cov)
- [ ] Add test utilities and fixtures
- [ ] Create test documentation
@@ -43,8 +43,8 @@
- [ ] Usage examples for each client API
- [ ] Installation guide
- [ ] Contributing guidelines
- [ ] README with quick start
- [x] README (cheatsheet)
---
**Last Updated:** 2026-02-07
**Last Updated:** 2026-09-30
+2
View File
@@ -0,0 +1,2 @@
target/
Cargo.lock
+18
View File
@@ -0,0 +1,18 @@
[package]
name = "resolvespec"
version = "0.1.0"
edition = "2021"
rust-version = "1.80"
description = "Client for ResolveSpec (JSON body) and FunctionSpec endpoints"
license = "MIT"
[dependencies]
reqwest = { version = "0.12", default-features = false, features = ["json", "rustls-tls"] }
serde = { version = "1", features = ["derive"] }
serde_json = "1"
base64 = "0.22"
thiserror = "1"
[dev-dependencies]
tokio = { version = "1", features = ["macros", "rt-multi-thread"] }
wiremock = "0.6"
+40
View File
@@ -0,0 +1,40 @@
# resolvespec (Rust)
Rust client for ResolveSpec (JSON body) and FunctionSpec. Async (`reqwest` + `tokio`). MSRV 1.80.
## Clients
| Type | Constructor | Methods |
|---|---|---|
| `ResolveSpecClient` | `new(base_url)` / `from_builder(ClientBuilder)` | `get_metadata` `read` `create` `update` `delete` |
| `FuncSpecClient` | `new(base_url)` / `from_builder(ClientBuilder)` | `query` `query_list` `request` |
`ClientBuilder::new(url).token().header().timeout().http_client()`. Precedence: Content-Type < custom headers < bearer token.
## ResolveSpec
- `RecordId`: `Int`/`Str` → URL, `Many(Vec<String>)` → body (`From` impls provided).
- `Options` (`Default` + struct update), optional fields are `Option`/empty `Vec`.
- Result: `Response{success, data: serde_json::Value, metadata}`; `resp.decode::<T>()`.
## FunctionSpec
- Routes are server-defined: pass the `path`.
- `Params = BTreeMap<String, Param>` → query string (`Param::List` → repeated keys).
- `FuncSpecOptions` → `X-*` headers: `filters`, `search_filters`, `custom_sql_where`, `custom_sql_or`, `sort`, `limit`, `offset`, `distinct`, `skip_count`, `skip_cache`, `response_format`.
- `query_list` fills `metadata` from `Content-Range`; 206 is success.
## Server quirks
- `sort` is raw SQL in ORDER BY (client sends `col ASC|DESC`).
- One search operator per column.
- Values starting `ZIP_` / `__` are base64-decoded by the server.
- Non-ASCII, control chars and edge spaces are auto-encoded (`ZIP_`).
## Errors
`Error::Api { status, message, error: ApiError{code, message, detail, sql} }`, `Error::Http`, `Error::Json`.
## Test
`cargo test`
+90
View File
@@ -0,0 +1,90 @@
use std::collections::HashMap;
use std::time::Duration;
use reqwest::header::{HeaderMap, HeaderName, HeaderValue, AUTHORIZATION, CONTENT_TYPE};
use crate::error::Result;
/// Shared HTTP configuration.
#[derive(Clone)]
pub(crate) struct Config {
pub base_url: String,
pub token: Option<String>,
pub headers: HashMap<String, String>,
pub http: reqwest::Client,
}
/// Builder options shared by both clients.
#[derive(Default, Clone)]
pub struct ClientBuilder {
base_url: String,
token: Option<String>,
headers: HashMap<String, String>,
timeout: Option<Duration>,
http: Option<reqwest::Client>,
}
impl ClientBuilder {
pub fn new(base_url: &str) -> Self {
Self { base_url: base_url.trim_end_matches('/').into(), timeout: Some(Duration::from_secs(30)), ..Default::default() }
}
pub fn token(mut self, token: &str) -> Self {
self.token = Some(token.into());
self
}
pub fn header(mut self, name: &str, value: &str) -> Self {
self.headers.insert(name.into(), value.into());
self
}
pub fn timeout(mut self, t: Duration) -> Self {
self.timeout = Some(t);
self
}
pub fn http_client(mut self, c: reqwest::Client) -> Self {
self.http = Some(c);
self
}
pub(crate) fn config(self) -> Result<Config> {
let http = match self.http {
Some(c) => c,
None => {
let mut b = reqwest::Client::builder();
if let Some(t) = self.timeout {
b = b.timeout(t);
}
b.build()?
}
};
Ok(Config { base_url: self.base_url, token: self.token, headers: self.headers, http })
}
}
impl Config {
/// Content-Type < custom headers < extra (per-call) < bearer token.
pub fn headers(&self, extra: &HashMap<String, String>) -> HeaderMap {
let mut m = HeaderMap::new();
m.insert(CONTENT_TYPE, HeaderValue::from_static("application/json"));
for (k, v) in self.headers.iter().chain(extra.iter()) {
if let (Ok(n), Ok(v)) = (HeaderName::try_from(k.as_str()), HeaderValue::from_str(v)) {
m.insert(n, v);
}
}
if let Some(t) = &self.token {
if let Ok(v) = HeaderValue::from_str(&format!("Bearer {t}")) {
m.insert(AUTHORIZATION, v);
}
}
m
}
}
pub(crate) fn path_segment(s: &str) -> String {
let mut out = String::new();
for b in s.bytes() {
match b {
b'A'..=b'Z' | b'a'..=b'z' | b'0'..=b'9' | b'-' | b'.' | b'_' | b'~' => out.push(b as char),
_ => out.push_str(&format!("%{b:02X}")),
}
}
out
}
+34
View File
@@ -0,0 +1,34 @@
use crate::types::ApiError;
/// Returned on transport failure, a non-2xx response or an unsuccessful API result.
#[derive(Debug, thiserror::Error)]
pub enum Error {
#[error("{message}")]
Api { status: u16, message: String, error: ApiError },
#[error(transparent)]
Http(#[from] reqwest::Error),
#[error(transparent)]
Json(#[from] serde_json::Error),
}
pub type Result<T> = std::result::Result<T, Error>;
pub(crate) fn error_from(status: u16, body: &str) -> Error {
let parsed: Option<serde_json::Value> = serde_json::from_str(body).ok();
let err: ApiError = parsed
.as_ref()
.and_then(|v| v.get("error"))
.and_then(|e| serde_json::from_value(e.clone()).ok())
.unwrap_or_default();
let message = if !err.message.is_empty() {
err.message.clone()
} else {
let text = if parsed.is_none() { body.trim().chars().take(200).collect::<String>() } else { String::new() };
if text.is_empty() {
format!("{} ({})", reqwest::StatusCode::from_u16(status).ok().and_then(|s| s.canonical_reason()).unwrap_or("Error"), status)
} else {
text
}
};
Error::Api { status, message, error: err }
}
+263
View File
@@ -0,0 +1,263 @@
use std::collections::{BTreeMap, HashMap};
use base64::{engine::general_purpose::STANDARD, Engine};
use reqwest::Method;
use serde_json::Value;
use crate::client::{ClientBuilder, Config};
use crate::error::{error_from, Result};
use crate::types::{FilterOption, Metadata, Response, SortOption};
/// Options sent to funcspec endpoints as `X-*` headers.
///
/// Server behaviour (`pkg/funcspec`): `sort` is inserted raw into ORDER BY (so it is sent as SQL
/// terms); only one search operator per column is kept; values starting with `ZIP_` or `__`
/// are base64-decoded by the server, so such plaintext values cannot be sent faithfully.
#[derive(Debug, Clone, Default)]
pub struct FuncSpecOptions {
/// eq+AND -> X-FieldFilter; others X-SearchOp / X-SearchOr.
pub filters: Vec<FilterOption>,
/// X-SearchFilter-{col}: text ILIKE.
pub search_filters: BTreeMap<String, String>,
pub custom_sql_where: Option<String>,
pub custom_sql_or: Option<String>,
pub sort: Vec<SortOption>,
pub limit: Option<i64>,
pub offset: Option<i64>,
pub distinct: Option<bool>,
pub skip_count: Option<bool>,
pub skip_cache: Option<bool>,
/// simple | detail | syncfusion
pub response_format: Option<String>,
}
/// Query-string parameter value. `List` is sent as repeated keys (server: IN filter).
#[derive(Debug, Clone)]
pub enum Param {
Str(String),
Int(i64),
Bool(bool),
List(Vec<String>),
}
impl From<&str> for Param {
fn from(v: &str) -> Self {
Self::Str(v.into())
}
}
impl From<String> for Param {
fn from(v: String) -> Self {
Self::Str(v)
}
}
impl From<i64> for Param {
fn from(v: i64) -> Self {
Self::Int(v)
}
}
impl From<bool> for Param {
fn from(v: bool) -> Self {
Self::Bool(v)
}
}
impl From<Vec<String>> for Param {
fn from(v: Vec<String>) -> Self {
Self::List(v)
}
}
pub type Params = BTreeMap<String, Param>;
fn operator(op: &str) -> &str {
match op {
"eq" => "equals",
"neq" => "notequals",
"gt" => "greaterthan",
"gte" => "greaterthanorequal",
"lt" => "lessthan",
"lte" => "lessthanorequal",
"like" | "ilike" | "contains" => "contains",
"startswith" => "beginswith",
"endswith" => "endswith",
"in" => "in",
"between" => "between",
"between_inclusive" => "betweeninclusive",
"is_null" => "empty",
"is_not_null" => "notempty",
other => other,
}
}
fn scalar(v: &Value) -> String {
match v {
Value::Null => String::new(),
Value::String(s) => s.clone(),
Value::Array(a) => a.iter().map(scalar).collect::<Vec<_>>().join(","),
other => other.to_string(),
}
}
/// Base64 (UTF-8) with the `ZIP_` prefix.
pub fn encode_header_value(v: &str) -> String {
format!("ZIP_{}", STANDARD.encode(v.as_bytes()))
}
/// Decode a value that may carry a `ZIP_` or `__` prefix (nested allowed).
pub fn decode_header_value(v: &str) -> String {
for p in ["ZIP_", "__"] {
if let Some(rest) = v.strip_prefix(p) {
let mut b64: String = rest.chars().filter(|c| !matches!(c, '\n' | '\r' | ' ')).collect();
while b64.len() % 4 != 0 {
b64.push('=');
}
return match STANDARD.decode(b64).ok().and_then(|b| String::from_utf8(b).ok()) {
Some(s) => decode_header_value(&s),
None => v.to_string(),
};
}
}
v.to_string()
}
/// Encode values that are unsafe as raw header/query text (non-ASCII, control chars, edge spaces).
fn safe(v: &str) -> String {
if v != v.trim() || v.chars().any(|c| !c.is_ascii() || c.is_ascii_control()) {
encode_header_value(v)
} else {
v.to_string()
}
}
/// Build the `X-*` headers understood by `funcspec.ParseParameters`.
pub fn build_headers(o: &FuncSpecOptions) -> BTreeMap<String, String> {
let mut h = BTreeMap::new();
for f in &o.filters {
let logic = f.logic_operator.as_deref().unwrap_or("AND");
let v = safe(&scalar(&f.value));
if f.operator == "eq" && logic == "AND" {
h.insert(format!("X-FieldFilter-{}", f.column), v);
} else {
let kind = if logic == "OR" { "X-SearchOr" } else { "X-SearchOp" };
h.insert(format!("{kind}-{}-{}", operator(&f.operator), f.column), v);
}
}
for (col, text) in &o.search_filters {
h.insert(format!("X-SearchFilter-{col}"), safe(text));
}
if let Some(v) = o.custom_sql_where.as_deref().filter(|s| !s.is_empty()) {
h.insert("X-Custom-SQL-W".into(), safe(v));
}
if let Some(v) = o.custom_sql_or.as_deref().filter(|s| !s.is_empty()) {
h.insert("X-Custom-SQL-Or".into(), safe(v));
}
if !o.sort.is_empty() {
let terms: Vec<String> = o
.sort
.iter()
.map(|s| format!("{} {}", s.column, if s.direction.eq_ignore_ascii_case("desc") { "DESC" } else { "ASC" }))
.collect();
h.insert("X-Sort".into(), safe(&terms.join(","))); // funcspec puts this verbatim into ORDER BY
}
if let Some(n) = o.limit {
h.insert("X-Limit".into(), n.to_string());
}
if let Some(n) = o.offset {
h.insert("X-Offset".into(), n.to_string());
}
for (name, v) in [("X-Distinct", o.distinct), ("X-SkipCount", o.skip_count), ("X-SkipCache", o.skip_cache)] {
if let Some(b) = v {
h.insert(name.into(), b.to_string());
}
}
match o.response_format.as_deref() {
Some("simple") => h.insert("X-SimpleApi".into(), "true".into()),
Some("detail") => h.insert("X-DetailApi".into(), "true".into()),
Some("syncfusion") => h.insert("X-Syncfusion".into(), "true".into()),
_ => None,
};
h
}
/// Build query-string pairs: bools -> true/false, lists -> repeated keys.
pub fn build_query(p: &Params) -> Vec<(String, String)> {
let mut out = Vec::new();
for (k, v) in p {
match v {
Param::Str(s) => out.push((k.clone(), safe(s))),
Param::Int(n) => out.push((k.clone(), n.to_string())),
Param::Bool(b) => out.push((k.clone(), b.to_string())),
Param::List(l) => out.extend(l.iter().map(|s| (k.clone(), safe(s)))),
}
}
out
}
fn metadata(content_range: Option<&str>, limit: Option<i64>) -> Metadata {
let mut m = Metadata { limit: limit.unwrap_or(0), ..Default::default() };
if let Some(cr) = content_range {
// "items {start}-{end}/{total}"
let rest = cr.rsplit(' ').next().unwrap_or("");
if let Some((range, total)) = rest.split_once('/') {
if let (Some((s, e)), Ok(t)) = (range.split_once('-'), total.parse::<i64>()) {
if let (Ok(s), Ok(e)) = (s.parse::<i64>(), e.parse::<i64>()) {
m.total = t;
m.filtered = t;
m.count = e - s;
m.offset = s;
}
}
}
}
m
}
/// Client for user-defined SQL endpoints. Routes are defined by the server application.
#[derive(Clone)]
pub struct FuncSpecClient {
cfg: Config,
}
impl FuncSpecClient {
pub fn new(base_url: &str) -> Result<Self> {
Self::from_builder(ClientBuilder::new(base_url))
}
pub fn from_builder(b: ClientBuilder) -> Result<Self> {
Ok(Self { cfg: b.config()? })
}
async fn call(&self, method: Method, path: &str, params: &Params, options: Option<&FuncSpecOptions>, list: bool) -> Result<Response> {
let url = format!("{}/{}", self.cfg.base_url, path.trim_start_matches('/'));
let extra: HashMap<String, String> = options.map(|o| build_headers(o).into_iter().collect()).unwrap_or_default();
let resp = self.cfg.http.request(method, url).headers(self.cfg.headers(&extra)).query(&build_query(params)).send().await?;
let status = resp.status();
let cr = resp.headers().get("content-range").and_then(|v| v.to_str().ok()).map(str::to_owned);
let text = resp.text().await?;
if !status.is_success() {
// 206 Partial Content is success
return Err(error_from(status.as_u16(), &text));
}
let data = if text.trim().is_empty() { Value::Null } else { serde_json::from_str(&text)? };
Ok(Response {
success: true,
data,
metadata: list.then(|| metadata(cr.as_deref(), options.and_then(|o| o.limit))),
error: None,
})
}
/// Single-record endpoint (`SqlQuery`). `data` is the row object.
pub async fn query(&self, path: &str, params: &Params, options: Option<&FuncSpecOptions>) -> Result<Response> {
self.call(Method::GET, path, params, options, false).await
}
/// List endpoint (`SqlQueryList`). Metadata comes from Content-Range.
pub async fn query_list(&self, path: &str, params: &Params, options: Option<&FuncSpecOptions>) -> Result<Response> {
self.call(Method::GET, path, params, options, true).await
}
/// Like `query` / `query_list` with an explicit HTTP method (routes are app-defined).
pub async fn request(&self, method: Method, path: &str, params: &Params, options: Option<&FuncSpecOptions>, list: bool) -> Result<Response> {
self.call(method, path, params, options, list).await
}
}

Some files were not shown because too many files have changed in this diff Show More