mirror of
https://github.com/bitechdev/ResolveSpec.git
synced 2026-10-01 12:31:59 +00:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
ec8d4d2c77 | ||
|
|
a3287f3b53 | ||
|
|
1e4a76643d | ||
|
|
a81031b83d | ||
|
|
ac4cf9b4b6 | ||
|
|
b35399fdfa | ||
|
|
a65ca5f5ce | ||
|
|
5933637a88 | ||
|
|
7f84debdc5 | ||
|
|
2042205817 | ||
|
|
6335bfe87e | ||
|
|
129c1a043d | ||
|
|
da0b1f5123 | ||
|
|
ff76eb8e1f | ||
|
|
3b93802a25 | ||
|
|
6b6f540ab0 | ||
|
|
4cbe4f597d | ||
|
|
ed457eb14a | ||
|
|
97bcb44fdc | ||
|
|
17bb6ea76d | ||
|
|
ce706bacda | ||
|
|
eb492d52aa | ||
|
|
f2dbe2561c | ||
|
|
cd96404cdd | ||
|
|
b2b815552f | ||
|
|
f6a9daa89e | ||
|
|
54e6a3b17c | ||
|
|
ab3d2b5b04 | ||
|
|
9c4d916490 | ||
|
|
3e327d0c78 | ||
|
|
62cc14c02a | ||
|
|
4c5dffc3d1 | ||
|
|
2898b335f8 | ||
|
|
20ba8ed112 | ||
|
|
1214f69e0c | ||
|
|
89a58ab3a0 | ||
|
|
48081b4aa4 | ||
|
|
467dbd66c8 | ||
|
|
178d40587d | ||
|
|
d3a99550d9 | ||
|
|
8a94d884e7 | ||
|
|
f9c948ca4e | ||
|
|
164ba2b240 | ||
|
|
97fe88b3a6 | ||
|
|
a4e1abc1df | ||
|
|
d7cb111496 | ||
|
|
f66930c3c9 | ||
|
|
9533c3a0ed | ||
|
|
652621a70e | ||
|
|
c7b4530689 | ||
|
|
e1cf72834e | ||
|
|
3657aa94cc | ||
|
|
da1af1487e | ||
|
|
bc8bff7955 | ||
|
|
a74eebc7f3 | ||
|
|
6687a7a5cd | ||
|
|
e8fbbede7e | ||
|
|
7f8982fa35 | ||
|
|
b587cbd3c4 | ||
|
|
a220338eea | ||
|
|
20c67166d0 | ||
|
|
749dad4ed1 | ||
|
|
d6c5740f9c | ||
|
|
817b781c88 | ||
|
|
87eaa9e18c | ||
|
|
4f6878099b | ||
|
|
0d8b136b91 | ||
|
|
6de9be0ae7 | ||
|
|
82e923b16e | ||
|
|
9a664593f0 | ||
|
|
6e3124e4e0 | ||
|
|
d5de48011b | ||
|
|
6bd6a6f164 |
@@ -27,6 +27,17 @@ jobs:
|
|||||||
with:
|
with:
|
||||||
name: coverage-report
|
name: coverage-report
|
||||||
path: coverage.html
|
path: coverage.html
|
||||||
|
race-tests:
|
||||||
|
name: Race Detector
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
steps:
|
||||||
|
- uses: actions/checkout@v6
|
||||||
|
- name: Set up Go
|
||||||
|
uses: actions/setup-go@v6
|
||||||
|
with:
|
||||||
|
go-version: "1.24"
|
||||||
|
- name: Run unit tests with the race detector
|
||||||
|
run: go test -race -count=1 ./pkg/...
|
||||||
integration-tests:
|
integration-tests:
|
||||||
name: Integration Tests
|
name: Integration Tests
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
|
|||||||
+1
-1
@@ -28,5 +28,5 @@ test.db
|
|||||||
/testserver
|
/testserver
|
||||||
tests/data/
|
tests/data/
|
||||||
node_modules/
|
node_modules/
|
||||||
resolvespec-js/dist/
|
clients/resolvespec-js/dist/
|
||||||
.codex
|
.codex
|
||||||
|
|||||||
@@ -30,6 +30,7 @@
|
|||||||
"linters": {
|
"linters": {
|
||||||
"enable": [
|
"enable": [
|
||||||
"gocritic",
|
"gocritic",
|
||||||
|
"gosec",
|
||||||
"misspell",
|
"misspell",
|
||||||
"revive"
|
"revive"
|
||||||
],
|
],
|
||||||
|
|||||||
@@ -1,11 +1,21 @@
|
|||||||
.PHONY: test test-unit test-integration docker-up docker-down clean
|
# Container compose command: podman if installed, else docker
|
||||||
|
COMPOSE ?= $(shell command -v podman >/dev/null 2>&1 && echo "podman compose" || echo "docker compose")
|
||||||
|
|
||||||
|
.PHONY: testserver-up testserver-down testserver-smoke test test-unit test-race test-integration docker-up docker-down clean
|
||||||
|
|
||||||
GOLANGCI_LINT := $(shell go env GOPATH)/bin/golangci-lint
|
GOLANGCI_LINT := $(shell go env GOPATH)/bin/golangci-lint
|
||||||
|
|
||||||
# Run all unit tests
|
# Run all unit tests
|
||||||
test-unit:
|
test-unit:
|
||||||
@echo "Running unit tests..."
|
@echo "Running unit tests..."
|
||||||
@go test ./pkg/resolvespec ./pkg/restheadspec -v -cover
|
@go test ./pkg/... -v -cover
|
||||||
|
|
||||||
|
# Run all unit tests under the race detector (kept separate from coverage:
|
||||||
|
# race builds are 2-10x slower). Only races on executed paths are reported,
|
||||||
|
# so this covers every package rather than a subset.
|
||||||
|
test-race:
|
||||||
|
@echo "Running unit tests with the race detector..."
|
||||||
|
@go test -race -count=1 ./pkg/...
|
||||||
|
|
||||||
# Run all integration tests (requires PostgreSQL)
|
# Run all integration tests (requires PostgreSQL)
|
||||||
test-integration:
|
test-integration:
|
||||||
@@ -13,7 +23,7 @@ test-integration:
|
|||||||
@go test -tags=integration ./pkg/resolvespec ./pkg/restheadspec -v
|
@go test -tags=integration ./pkg/resolvespec ./pkg/restheadspec -v
|
||||||
|
|
||||||
# Run all tests (unit + integration)
|
# Run all tests (unit + integration)
|
||||||
test: test-unit test-integration
|
test: test-unit test-race test-integration
|
||||||
|
|
||||||
release-version: ## Create and push a release with specific version (use: make release-version VERSION=v1.2.3 or make release-version to auto-increment)
|
release-version: ## Create and push a release with specific version (use: make release-version VERSION=v1.2.3 or make release-version to auto-increment)
|
||||||
@if [ -z "$(VERSION)" ]; then \
|
@if [ -z "$(VERSION)" ]; then \
|
||||||
@@ -75,7 +85,7 @@ lintfix: ## Run linter
|
|||||||
# Start PostgreSQL for integration tests
|
# Start PostgreSQL for integration tests
|
||||||
docker-up:
|
docker-up:
|
||||||
@echo "Starting PostgreSQL container..."
|
@echo "Starting PostgreSQL container..."
|
||||||
@podman compose up -d postgres-test
|
@$(COMPOSE) up -d postgres-test
|
||||||
@echo "Waiting for PostgreSQL to be ready..."
|
@echo "Waiting for PostgreSQL to be ready..."
|
||||||
@sleep 5
|
@sleep 5
|
||||||
@echo "PostgreSQL is ready!"
|
@echo "PostgreSQL is ready!"
|
||||||
@@ -83,12 +93,23 @@ docker-up:
|
|||||||
# Stop PostgreSQL container
|
# Stop PostgreSQL container
|
||||||
docker-down:
|
docker-down:
|
||||||
@echo "Stopping PostgreSQL container..."
|
@echo "Stopping PostgreSQL container..."
|
||||||
@podman compose down
|
@$(COMPOSE) down
|
||||||
|
|
||||||
|
# Test server + PostgreSQL in containers (dbtrace enabled)
|
||||||
|
|
||||||
|
testserver-up:
|
||||||
|
@$(COMPOSE) up -d --build postgres-test testserver
|
||||||
|
|
||||||
|
testserver-down:
|
||||||
|
@$(COMPOSE) down
|
||||||
|
|
||||||
|
testserver-smoke:
|
||||||
|
@COMPOSE="$(COMPOSE)" scripts/testserver-smoke.sh
|
||||||
|
|
||||||
# Clean up Docker volumes and test data
|
# Clean up Docker volumes and test data
|
||||||
clean:
|
clean:
|
||||||
@echo "Cleaning up..."
|
@echo "Cleaning up..."
|
||||||
@podman compose down -v
|
@$(COMPOSE) down -v
|
||||||
@echo "Cleanup complete!"
|
@echo "Cleanup complete!"
|
||||||
|
|
||||||
# Run integration tests with Docker (full workflow)
|
# Run integration tests with Docker (full workflow)
|
||||||
@@ -113,7 +134,8 @@ coverage-integration:
|
|||||||
|
|
||||||
help:
|
help:
|
||||||
@echo "Available targets:"
|
@echo "Available targets:"
|
||||||
@echo " test-unit - Run unit tests"
|
@echo " test-unit - Run unit tests for all packages (./pkg/...)"
|
||||||
|
@echo " test-race - Run unit tests for all packages with -race"
|
||||||
@echo " test-integration - Run integration tests (requires PostgreSQL)"
|
@echo " test-integration - Run integration tests (requires PostgreSQL)"
|
||||||
@echo " test - Run all tests"
|
@echo " test - Run all tests"
|
||||||
@echo " docker-up - Start PostgreSQL container"
|
@echo " docker-up - Start PostgreSQL container"
|
||||||
|
|||||||
@@ -13,7 +13,7 @@ ResolveSpec is a flexible and powerful REST API specification and implementation
|
|||||||
|
|
||||||
All share the same core architecture and provide dynamic data querying, relationship preloading, and complex filtering.
|
All share the same core architecture and provide dynamic data querying, relationship preloading, and complex filtering.
|
||||||
|
|
||||||

|
|
||||||
|
|
||||||
## Table of Contents
|
## Table of Contents
|
||||||
|
|
||||||
@@ -43,6 +43,7 @@ All share the same core architecture and provide dynamic data querying, relation
|
|||||||
* **Pagination**: Built-in limit/offset and cursor-based pagination (both ResolveSpec and RestHeadSpec)
|
* **Pagination**: Built-in limit/offset and cursor-based pagination (both ResolveSpec and RestHeadSpec)
|
||||||
* **Computed Columns**: Define virtual columns for complex calculations
|
* **Computed Columns**: Define virtual columns for complex calculations
|
||||||
* **Custom Operators**: Add custom SQL conditions when needed
|
* **Custom Operators**: Add custom SQL conditions when needed
|
||||||
|
* **🆕 One Transaction Per Request**: Every statement and DB-touching hook of a request runs on one transaction; `OnTxBegin` hook stamps transaction-local settings (RLS) first. See [pkg/common/TRANSACTIONS.md](pkg/common/TRANSACTIONS.md)
|
||||||
* **🆕 Recursive CRUD Handler**: Automatically handle nested object graphs with foreign key resolution and per-record operation control via `_request` field
|
* **🆕 Recursive CRUD Handler**: Automatically handle nested object graphs with foreign key resolution and per-record operation control via `_request` field
|
||||||
|
|
||||||
### Architecture (v2.0+)
|
### Architecture (v2.0+)
|
||||||
@@ -370,6 +371,14 @@ ResolveSpec is designed for testability with mockable interfaces. For testing ex
|
|||||||
- [RestHeadSpec Testing](pkg/restheadspec/README.md#testing)
|
- [RestHeadSpec Testing](pkg/restheadspec/README.md#testing)
|
||||||
- [WebSocketSpec Testing](pkg/websocketspec/README.md)
|
- [WebSocketSpec Testing](pkg/websocketspec/README.md)
|
||||||
|
|
||||||
|
### Test Server (dbtrace, real PostgreSQL)
|
||||||
|
|
||||||
|
* `make testserver-up` / `make testserver-down`: testserver + PostgreSQL via compose (host networking)
|
||||||
|
* `make testserver-smoke`: create, read, update, delete, batch create/delete against the testserver
|
||||||
|
* Ports: testserver `8123`, PostgreSQL `8124`
|
||||||
|
* `dbtrace` logs per request `tx`, `tx_queries`, `pooled`, `raw`; `pooled=0` is the target
|
||||||
|
* Integration tests default to PostgreSQL on `localhost:8124`
|
||||||
|
|
||||||
## Continuous Integration
|
## Continuous Integration
|
||||||
|
|
||||||
ResolveSpec uses GitHub Actions for automated testing and quality checks. The CI pipeline runs on every push and pull request.
|
ResolveSpec uses GitHub Actions for automated testing and quality checks. The CI pipeline runs on every push and pull request.
|
||||||
@@ -482,6 +491,19 @@ Execute SQL functions and queries through a simple HTTP API with header-based pa
|
|||||||
|
|
||||||
For complete documentation, see [pkg/funcspec/](pkg/funcspec/).
|
For complete documentation, see [pkg/funcspec/](pkg/funcspec/).
|
||||||
|
|
||||||
|
#### Clients
|
||||||
|
|
||||||
|
All clients are under [clients/](clients/README.md); wire behaviour is identical across them.
|
||||||
|
|
||||||
|
| Client | Language | Specs | Docs |
|
||||||
|
|---|---|---|---|
|
||||||
|
| `resolvespec-js` | TypeScript | ResolveSpec, HeaderSpec, FunctionSpec, WebSocketSpec | [README](clients/resolvespec-js/README.md) |
|
||||||
|
| `resolvespec-python` | Python >= 3.11 | ResolveSpec, HeaderSpec, FunctionSpec, WebSocketSpec | [README](clients/resolvespec-python/README.md) |
|
||||||
|
| `resolvespec-go` | Go | ResolveSpec, FunctionSpec | [README](clients/resolvespec-go/README.md) |
|
||||||
|
| `resolvespec-rs` | Rust | ResolveSpec, FunctionSpec | [README](clients/resolvespec-rs/README.md) |
|
||||||
|
| `resolvespec-cs` | C# (.NET 8) | ResolveSpec, FunctionSpec | [README](clients/resolvespec-cs/README.md) |
|
||||||
|
| `resolvespec-dart` | Dart / Flutter | ResolveSpec, FunctionSpec | [README](clients/resolvespec-dart/README.md) |
|
||||||
|
|
||||||
#### ResolveSpec JS - TypeScript Client Library
|
#### ResolveSpec JS - TypeScript Client Library
|
||||||
|
|
||||||
TypeScript/JavaScript client library supporting all three REST and WebSocket protocols.
|
TypeScript/JavaScript client library supporting all three REST and WebSocket protocols.
|
||||||
@@ -491,7 +513,7 @@ TypeScript/JavaScript client library supporting all three REST and WebSocket pro
|
|||||||
- Header-based REST client (`HeaderSpecClient`)
|
- Header-based REST client (`HeaderSpecClient`)
|
||||||
- WebSocket client (`WebSocketClient`) with CRUD, subscriptions, heartbeat, reconnect
|
- WebSocket client (`WebSocketClient`) with CRUD, subscriptions, heartbeat, reconnect
|
||||||
|
|
||||||
For complete documentation, see [resolvespec-js/README.md](resolvespec-js/README.md).
|
For complete documentation, see [clients/resolvespec-js/README.md](clients/resolvespec-js/README.md).
|
||||||
|
|
||||||
### Real-Time Communication
|
### Real-Time Communication
|
||||||
|
|
||||||
@@ -576,10 +598,31 @@ Centralized management of multiple database connections with support for Postgre
|
|||||||
- Multiple named database connections
|
- Multiple named database connections
|
||||||
- Multi-ORM access (Bun, GORM, Native SQL) sharing the same connection pool
|
- Multi-ORM access (Bun, GORM, Native SQL) sharing the same connection pool
|
||||||
- Automatic SQLite schema translation (`schema.table` → `schema_table`)
|
- Automatic SQLite schema translation (`schema.table` → `schema_table`)
|
||||||
- Health checks with auto-reconnect
|
- Background health checks (report status; they never close the pool)
|
||||||
- Prometheus metrics for monitoring
|
- Prometheus metrics for monitoring
|
||||||
- Configuration-driven via YAML
|
- Configuration-driven via YAML
|
||||||
- Per-connection statistics and management
|
- Per-connection statistics and management, including pool limit (`max`) and total connections ever opened (`dbmanager_connection_pool_size{state="max"}`, `dbmanager_connections_opened_total`)
|
||||||
|
|
||||||
|
**How to use it correctly**:
|
||||||
|
|
||||||
|
```go
|
||||||
|
mgr, err := dbmanager.NewManager(cfg) // or dbmanager.SetupManager(cfg) + GetInstance()
|
||||||
|
if err != nil { /* handle */ }
|
||||||
|
if err := mgr.Connect(ctx); err != nil { /* handle */ } // SetupManager does NOT connect
|
||||||
|
defer mgr.Close() // once, at shutdown
|
||||||
|
|
||||||
|
conn, _ := mgr.GetDefault()
|
||||||
|
db, _ := conn.Bun() // or conn.GORM() / conn.Native() / conn.Database()
|
||||||
|
handler := restheadspec.NewHandlerWithBun(db)
|
||||||
|
```
|
||||||
|
|
||||||
|
- **Fetch a handle once and keep it.** `Bun()`, `GORM()`, `Native()` and `Database()` return handles over one long-lived `*sql.DB`. You do not need to re-fetch them per request, and they stay valid for the life of the connection.
|
||||||
|
- **Never close a handle yourself.** Closing a `*bun.DB`, `*gorm.DB` or the `*sql.DB` closes the shared pool for everyone. Only `mgr.Close()` (at shutdown) should close it. After `Close`, the handles are dead.
|
||||||
|
- **Don't reconnect to recover from errors.** `database/sql` already discards bad connections and dials new ones. The manager does not close the pool on errors or failed health checks. `conn.Reconnect(ctx)` is for explicit operator use only (for example after rotating credentials): on PostgreSQL it retires pooled connections without closing the pool, so held handles keep working. Other databases close and reopen the pool, which invalidates handles you already hold.
|
||||||
|
- **Bring your own `*sql.DB`.** `dbmanager.NewConnectionFromDB(name, type, db)` wraps a pool you opened. The manager never closes it (`Close` only logs a warning); you own it and must close it.
|
||||||
|
- **Set deadlines on request contexts.** `query_timeout` is applied to PostgreSQL as `statement_timeout` (server side) and TCP timeouts detect dead sockets, but pass a context with a deadline to your queries so callers fail fast.
|
||||||
|
- **Pool tuning.** Keep `conn_max_idle_time` below the shortest idle timeout of any NAT, load balancer or pgbouncer between you and the database (typically 60-240s). SQLite `:memory:` is pinned to a single connection.
|
||||||
|
- **Health checks** run every `health_check_interval` (default 15s; a negative value disables them) and publish Prometheus metrics. `enable_auto_reconnect` is deprecated and ignored.
|
||||||
|
|
||||||
For documentation, see [pkg/dbmanager/README.md](pkg/dbmanager/README.md).
|
For documentation, see [pkg/dbmanager/README.md](pkg/dbmanager/README.md).
|
||||||
|
|
||||||
@@ -597,7 +640,15 @@ For documentation, see [pkg/security/README.md](pkg/security/README.md) (see "Di
|
|||||||
|
|
||||||
#### Middleware
|
#### Middleware
|
||||||
|
|
||||||
HTTP middleware collection for common tasks (CORS, logging, metrics, etc.).
|
HTTP middleware collection for common tasks (CORS, logging, metrics, rate limiting, etc.).
|
||||||
|
|
||||||
|
**Client request queue** (`middleware.ClientQueue`): limits how many requests each client runs concurrently and queues the rest first-in-first-out, smoothing bursts such as a page load that fires ~15 requests at once. Clients are identified by `X-Client-Id`, then `Authorization`, then the built-in session, then IP, so no client changes are required. Exposes burst, wait and queue-depth Prometheus metrics. Add it through the middleware slot of `SetupMuxRoutes` / `SetupBunRouterRoutes`:
|
||||||
|
|
||||||
|
```go
|
||||||
|
q := middleware.NewClientQueue(middleware.ClientQueueConfig{MaxConcurrent: 10})
|
||||||
|
defer q.Close()
|
||||||
|
restheadspec.SetupMuxRoutes(router, handler, middleware.Chain(authMiddleware, q.Middleware))
|
||||||
|
```
|
||||||
|
|
||||||
For documentation, see [pkg/middleware/README.md](pkg/middleware/README.md).
|
For documentation, see [pkg/middleware/README.md](pkg/middleware/README.md).
|
||||||
|
|
||||||
@@ -631,12 +682,29 @@ Configuration management with support for multiple formats and environments.
|
|||||||
|
|
||||||
For documentation, see [pkg/config/README.md](pkg/config/README.md).
|
For documentation, see [pkg/config/README.md](pkg/config/README.md).
|
||||||
|
|
||||||
|
#### DB Trace
|
||||||
|
|
||||||
|
Per-request DB call counting (`tx`, `tx_queries`, `pooled`, `raw`) and pool logging. Off by default.
|
||||||
|
|
||||||
|
For documentation, see [pkg/dbtrace/README.md](pkg/dbtrace/README.md).
|
||||||
|
|
||||||
|
### Core Libraries
|
||||||
|
|
||||||
|
| Package | Purpose |
|
||||||
|
|---|---|
|
||||||
|
| [`pkg/common`](pkg/common/) | Shared interfaces (database, request/response adapters), validation, recursive CRUD, request transactions ([TRANSACTIONS.md](pkg/common/TRANSACTIONS.md)) |
|
||||||
|
| [`pkg/modelregistry`](pkg/modelregistry/) | Model registration by schema/entity and per-model access rules |
|
||||||
|
| [`pkg/reflection`](pkg/reflection/) | Model/struct reflection helpers (primary keys, columns, relations) |
|
||||||
|
| [`pkg/spectypes`](pkg/spectypes/) | SQL-aware types (nullable, JSONB, PostGIS, vector) |
|
||||||
|
| [`pkg/logger`](pkg/logger/) | Logging used by all packages |
|
||||||
|
| [`pkg/testmodels`](pkg/testmodels/) | Shared test models and data for tests and the testserver |
|
||||||
|
|
||||||
## Security Considerations
|
## Security Considerations
|
||||||
|
|
||||||
* Implement proper authentication and authorization
|
* Implement proper authentication and authorization
|
||||||
* Validate all input parameters
|
* Validate all input parameters
|
||||||
* Use prepared statements (handled by GORM/Bun/your ORM)
|
* Use prepared statements (handled by GORM/Bun/your ORM)
|
||||||
* Implement rate limiting
|
* Implement rate limiting (`middleware.RateLimiter`) and per-client request queueing (`middleware.ClientQueue`)
|
||||||
* Control access at schema/entity level
|
* Control access at schema/entity level
|
||||||
* **New**: Database abstraction layer provides additional security through interface boundaries
|
* **New**: Database abstraction layer provides additional security through interface boundaries
|
||||||
|
|
||||||
@@ -654,6 +722,24 @@ This project is licensed under the MIT License - see the [LICENSE](LICENSE) file
|
|||||||
|
|
||||||
## What's New
|
## What's New
|
||||||
|
|
||||||
|
### Unreleased
|
||||||
|
|
||||||
|
**Single transaction per request**:
|
||||||
|
|
||||||
|
* **One tx per request**: hooks get the transaction in `hookCtx.Tx`, never the pool (`BeforeHandle` runs before any tx and must not touch the DB)
|
||||||
|
* **`OnTxBegin` hook**: all specs (mqttspec re-exports websocketspec's); fires once, first, in every tx; error or abort rolls back with no detail to the client
|
||||||
|
* **Second short tx**: create/update re-fetch, `BeforeScan` and post-commit hooks (`AfterCreate`, `AfterUpdate`, restheadspec `AfterRead`, funcspec `BeforeResponse`) run on a new tx after the first commits
|
||||||
|
* **Delete**: single and batch delete, hooks included, in one tx
|
||||||
|
* **websocketspec / mqttspec**: one tx per message; begin/commit failures answer `transaction_error`
|
||||||
|
* **resolvemcp**: read, create, update, delete transactional
|
||||||
|
* **RLS stamping**: `SecurityList.SetTxSettings(fn)`; `RegisterSecurityHooks` of every spec stamps `set_config(name, value, true)` on `OnTxBegin`; fails closed
|
||||||
|
* **New**: `common.RunRequestTx`, `common.TxContext`, `common.TxHookName`
|
||||||
|
* **Behavior changes**: `AfterDelete` failure now rolls the delete back; funcspec begin/commit failure answers 500 `transaction_error`
|
||||||
|
|
||||||
|
**Clients**: Go, Rust, C# and Dart clients for ResolveSpec and FunctionSpec under `clients/`.
|
||||||
|
|
||||||
|
**Test server**: compose uses host networking; ports `8123` (testserver) and `8124` (PostgreSQL), previously `8080` and `5434`.
|
||||||
|
|
||||||
### v3.2 (Latest - March 2026)
|
### v3.2 (Latest - March 2026)
|
||||||
|
|
||||||
**ResolveMCP - Model Context Protocol Server (🆕)**:
|
**ResolveMCP - Model Context Protocol Server (🆕)**:
|
||||||
@@ -773,3 +859,6 @@ This project is licensed under the MIT License - see the [LICENSE](LICENSE) file
|
|||||||
* Slogan generated using DALL-E
|
* Slogan generated using DALL-E
|
||||||
* AI used for documentation checking and correction
|
* AI used for documentation checking and correction
|
||||||
* Community feedback and contributions that made v2.0 and v2.1 possible
|
* Community feedback and contributions that made v2.0 and v2.1 possible
|
||||||
|
|
||||||
|
|
||||||
|

|
||||||
@@ -0,0 +1,145 @@
|
|||||||
|
# resolvemcp rewrite plan
|
||||||
|
|
||||||
|
Source: `audit/pkg/resolvemcp.audit.md`. Status: plan only, no code changed.
|
||||||
|
|
||||||
|
## Goal
|
||||||
|
|
||||||
|
Replace 4 tools + 1 resource per model with a fixed set of meta tools.
|
||||||
|
Endpoint guarded by OAuth / session token / API key; tools run as the authenticated caller.
|
||||||
|
Same rules as resolvespec CRUD, plus guardrails.
|
||||||
|
|
||||||
|
## Decisions
|
||||||
|
|
||||||
|
| Topic | Decision |
|
||||||
|
|---|---|
|
||||||
|
| Tools | Fixed meta tools; per-model tools/resources removed (breaking) |
|
||||||
|
| Functions | Explicit registry `Handler.RegisterFunction`; two kinds: Go callback (`func(ctx, tx, args)` + JSON-schema params) and SQL procedure by name (declared params); both behind `call_function`, run in tx with hooks |
|
||||||
|
| Create | `insert_into_table` included |
|
||||||
|
| Writes | update/delete by id **or** filters |
|
||||||
|
| Guardrails | require id or filters, max rows, `dry_run`, confirm token |
|
||||||
|
| Token scope | filter writes only; id writes = single row, no token |
|
||||||
|
| Confirm token store | in-memory, TTL, bound to user/table/filter hash; lost on restart, single instance |
|
||||||
|
| Read limits | server caps: limit, offset, batch, preload depth, timeout |
|
||||||
|
| Model exposure | all registered models visible; rules only restrict operations |
|
||||||
|
| Visibility | list tools show only what the caller may do (rules) |
|
||||||
|
| Identity | the authenticated caller's `UserContext`; **no fixed MCP user, no `SetUsername`, no service session, no background refresh** |
|
||||||
|
| Guard | endpoint always requires one of: OAuth bearer, session token, API key; no guest/optional mode |
|
||||||
|
| API key login | new `DatabaseAuthenticator.LoginWithAPIKey(ctx, rawKey)` + procedure `resolvespec_login_api_key`; validates key via keystore, creates session, returns `LoginResponse` |
|
||||||
|
| Session SQL | procedure mode + direct-SQL fallback (`ShouldUseProcedure`), same as `Login` |
|
||||||
|
| OAuth routes | `oauth2.go`/`oauth2_server.go` kept, part of the guard |
|
||||||
|
| Annotations | opt-in `Config.EnableAnnotations`, via `BeforeHandle` |
|
||||||
|
|
||||||
|
## Open
|
||||||
|
|
||||||
|
- None.
|
||||||
|
|
||||||
|
## Tools
|
||||||
|
|
||||||
|
| Tool | Purpose |
|
||||||
|
|---|---|
|
||||||
|
| `list_tables` | visible `schema.entity` + allowed ops |
|
||||||
|
| `describe_table` | columns, PK, relations, writable columns, rules, limits |
|
||||||
|
| `select_table` | filters, sort, columns, preloads, cursor; capped |
|
||||||
|
| `insert_into_table` | one or batch (capped); column allowlist |
|
||||||
|
| `update_table` | validated keys; id or filters; guardrails |
|
||||||
|
| `delete_from_table` | id or filters; guardrails |
|
||||||
|
| `list_functions` | registered functions + parameter schemas |
|
||||||
|
| `call_function` | validated args; tx + hooks + rules |
|
||||||
|
|
||||||
|
## Config additions
|
||||||
|
|
||||||
|
| Field | Purpose |
|
||||||
|
|---|---|
|
||||||
|
| `DefaultLimit`, `MaxLimit`, `MaxOffset` | read paging caps |
|
||||||
|
| `MaxBatch` | insert batch cap |
|
||||||
|
| `MaxPreloadDepth` | preload cap |
|
||||||
|
| `MaxWriteRows` | filter-write row cap |
|
||||||
|
| `QueryTimeout` | per-call context timeout |
|
||||||
|
| `ConfirmTTL` | confirm token lifetime |
|
||||||
|
| `EnableAnnotations` | opt-in annotate tool |
|
||||||
|
|
||||||
|
## Guardrail rules
|
||||||
|
|
||||||
|
| Rule | Behaviour |
|
||||||
|
|---|---|
|
||||||
|
| Target required | update/delete with neither id nor filters rejected |
|
||||||
|
| Max rows | count matches inside tx; abort above `MaxWriteRows` |
|
||||||
|
| `dry_run` | returns match count + preview, no write |
|
||||||
|
| Confirm token | filter write: first call returns token + preview; second call with token executes; bound to user, table, filter hash; expires at `ConfirmTTL` |
|
||||||
|
| Id write | single row, no token |
|
||||||
|
|
||||||
|
## Work items
|
||||||
|
|
||||||
|
### 1. API key login (`pkg/security`)
|
||||||
|
- Existing: keystore has `ValidateKey` and `KeyStoreAuthenticator`; `Login` needs a password; no key-to-session path.
|
||||||
|
- Add `resolvespec_login_api_key` to `SQLNames` (default + override) and a SQL script beside the existing procedures. Contract: `p_success, p_error, p_data`, input raw key; hashes, validates active/non-expired key, creates session for the key's user.
|
||||||
|
- Add `DatabaseAuthenticator.LoginWithAPIKey(ctx, rawKey)`; procedure first, direct-SQL fallback via `ShouldUseProcedure`.
|
||||||
|
- Hashed lookup; same generic error for unknown, expired or inactive key; no key material in logs.
|
||||||
|
- Expose through the chain/composite authenticators so the middleware can accept it.
|
||||||
|
|
||||||
|
### 2. Endpoint guard
|
||||||
|
- Wire `security.NewAuthMiddleware` with a chain of OAuth bearer, session token (header/cookie) and API key.
|
||||||
|
- `SetupMux*`/`SetupBunRouter*` helpers require the guard; unauthenticated serving only when explicitly constructed without it, logged loudly. Remove `OptionalAuth*` from the MCP path.
|
||||||
|
- Caller `UserContext` flows to every tool call context; rules, RLS and `OnTxBegin` apply to that user.
|
||||||
|
|
||||||
|
### 3. Security fixes (audit #1-6)
|
||||||
|
- Put model rules in request context (`withRequestData`) and/or `AddRegistry` on construction.
|
||||||
|
- Add `BeforeCreate` -> `CheckModelCreateAllowed` (new in `pkg/security`).
|
||||||
|
- Call `BeforeHandle` first in `executeUpdate`.
|
||||||
|
- Validate create/update keys against `ColumnValidator`; reject unknown; resolve column names from model, not json tags.
|
||||||
|
- Apply row security to update/delete pre-read; fail if row not visible.
|
||||||
|
- Annotate tool: opt-in + `BeforeHandle`.
|
||||||
|
|
||||||
|
### 4. Limits (audit #7, #13)
|
||||||
|
- Apply default/max limit, max offset, batch cap, preload depth cap, timeout.
|
||||||
|
- Validate preload names against model relations.
|
||||||
|
- Count only when requested.
|
||||||
|
|
||||||
|
### 5. Error and panic surface (audit #9, #17)
|
||||||
|
- Stable error codes + short message to client.
|
||||||
|
- Details and stack logged server-side.
|
||||||
|
- Recover hook panics.
|
||||||
|
|
||||||
|
### 6. Update/create semantics (audit #10-12)
|
||||||
|
- `SET` from validated incoming keys only; explicit null supported.
|
||||||
|
- Lock row (`FOR UPDATE`) on update.
|
||||||
|
- Refetch and `After*` hooks inside the same tx, or report committed write with warning if not possible (see `audit/single_tran.md`).
|
||||||
|
|
||||||
|
### 7. Smaller fixes (audit #8, #14, #15)
|
||||||
|
- SSE pool: require `BaseURL` or cap/evict; allowlist Host.
|
||||||
|
- Uniform not-found vs hook error text.
|
||||||
|
- Add mutex to `HookRegistry`.
|
||||||
|
|
||||||
|
### 8. Meta tools
|
||||||
|
- New file for meta tools; reuse parse helpers and `buildModelInfo` for `describe_table`.
|
||||||
|
- Remove per-model register functions and resources.
|
||||||
|
- `RegisterModel` only registers to registry.
|
||||||
|
- Function registry (Go callback kind + SQL procedure kind) + validation of args against declared schema.
|
||||||
|
|
||||||
|
### 9. Tests
|
||||||
|
- Update `tx_test.go` (calls `executeRead/Create/Update/Delete`) and `tools_test.go`.
|
||||||
|
- New: rule enforcement, unknown keys, limits, guardrails (cap, dry_run, token expiry/binding), guard rejects unauthenticated, API key login (valid, expired, inactive, unknown), visibility filtering, `-race`.
|
||||||
|
- Check for existing test data first; ask before generating any.
|
||||||
|
|
||||||
|
### 10. Docs
|
||||||
|
- Rewrite `pkg/resolvemcp/README.md` cheatsheet style.
|
||||||
|
- Document `resolvespec_login_api_key` in `pkg/security` docs.
|
||||||
|
- Update root README references.
|
||||||
|
- Update audit file when findings are closed.
|
||||||
|
|
||||||
|
## Order
|
||||||
|
|
||||||
|
1. `LoginWithAPIKey` + procedure in `pkg/security` (1)
|
||||||
|
2. Guard + security fixes (2-3)
|
||||||
|
3. Limits, errors, update/create semantics (4-6)
|
||||||
|
4. Meta tools + function registry (8)
|
||||||
|
5. Smaller fixes (7)
|
||||||
|
6. Tests (9), docs (10)
|
||||||
|
|
||||||
|
## Breaking changes
|
||||||
|
|
||||||
|
- Per-model tools and resources gone.
|
||||||
|
- MCP endpoint requires authentication.
|
||||||
|
- Annotate tool off by default.
|
||||||
|
- Update/create reject unknown keys.
|
||||||
|
- Reads capped by default.
|
||||||
@@ -0,0 +1,677 @@
|
|||||||
|
# Audit: cross-cutting findings across `pkg/*`
|
||||||
|
|
||||||
|
| | |
|
||||||
|
|---|---|
|
||||||
|
| **Scope** | all 23 packages under `pkg/` (64 065 non-test lines) |
|
||||||
|
| **Audit date** | 2026-09-29 |
|
||||||
|
| **Axes** | thread locking/waiting, slowness, security, panic handling & logging |
|
||||||
|
| **Threat model** | hostile internet client; request bodies, headers, query params, schema/table/column names all attacker-controlled |
|
||||||
|
|
||||||
|
This file records findings that are **not specific to one package** — they are
|
||||||
|
properties of the repository or patterns repeated across many packages. The
|
||||||
|
per-package audits reference this file rather than restating them.
|
||||||
|
|
||||||
|
## Findings
|
||||||
|
|
||||||
|
| # | Severity | Axis | Finding |
|
||||||
|
|---|---|---|---|
|
||||||
|
| X1 | **High** | locking | `-race` is never run anywhere; no package is ever race-checked |
|
||||||
|
| X2 | **High** | testing | `go test` runs against 2 of 23 packages; the other 21 are only compiled and vetted |
|
||||||
|
| X3 | **High** | testing | Every integration-test step is `continue-on-error: true` — integration failures cannot fail CI |
|
||||||
|
| X10 | **High** | security | Whole subsystems are declared, configured, documented and tested but never installed — including every protective middleware and the metrics provider |
|
||||||
|
| X4 | **Medium** | security | `gosec` is not enabled in `.golangci.json`; no SAST runs on a package set full of dynamic SQL |
|
||||||
|
| X5 | **Medium** | locking | Unsynchronized package-level mutable globals are the dominant concurrency pattern |
|
||||||
|
| X6 | **Medium** | security | Insecure-by-default transport across the board: `sslmode: disable`, `WithInsecure()`, no TLS in cache configs |
|
||||||
|
| X7 | **Medium** | panic handling | Panic handling is inconsistent and, where it exists, tends to fail open |
|
||||||
|
| X8 | **Medium** | security | `logger.Warn`/`Error` forward every message to Sentry unscrubbed, and error strings routinely embed attacker data *(partly fixed 2026-09-30: redaction and rate limiting added in `pkg/logger`; call sites still embed attacker data)* |
|
||||||
|
| X9 | **Low** | testing | Test coverage is extremely uneven: 5 packages have no test file at all |
|
||||||
|
|
||||||
|
The table is ordered by severity; the sections below are in ID order, since other
|
||||||
|
audit files reference these findings by number.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### X1. High — `-race` is never run
|
||||||
|
|
||||||
|
Verified by grep: the string `-race` does not appear in `Makefile`,
|
||||||
|
`.github/workflows/tests.yml`, `.github/workflows/maint.yml` or
|
||||||
|
`.github/workflows/make_tag.yml`.
|
||||||
|
|
||||||
|
Every test invocation in the repository:
|
||||||
|
|
||||||
|
```makefile
|
||||||
|
# Makefile:8
|
||||||
|
@go test ./pkg/resolvespec ./pkg/restheadspec -v -cover
|
||||||
|
# Makefile:13
|
||||||
|
@go test -tags=integration ./pkg/resolvespec ./pkg/restheadspec -v
|
||||||
|
# Makefile:97
|
||||||
|
@go test -tags=integration ./pkg/resolvespec ./pkg/restheadspec -v
|
||||||
|
# Makefile:103
|
||||||
|
@go test ./pkg/resolvespec ./pkg/restheadspec -coverprofile=coverage.out
|
||||||
|
# Makefile:110
|
||||||
|
@go test -tags=integration ./pkg/resolvespec ./pkg/restheadspec -coverprofile=coverage-integration.out
|
||||||
|
```
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
# .github/workflows/tests.yml — unit-tests job
|
||||||
|
- name: Run unit tests
|
||||||
|
run: go test ./pkg/resolvespec ./pkg/restheadspec -v -cover
|
||||||
|
```
|
||||||
|
|
||||||
|
**Why this matters.** This audit found unsynchronized concurrent access to
|
||||||
|
mutable state in **six** packages, and the Go race detector would have flagged
|
||||||
|
every one of them on the first run:
|
||||||
|
|
||||||
|
| Package | Racing state | Reference |
|
||||||
|
|---|---|---|
|
||||||
|
| `pkg/cache` | `defaultCache` read/written by concurrent request handlers | `cache.audit.md` finding 3 |
|
||||||
|
| `pkg/config` | `*viper.Viper` has no internal lock; `configInstance` singleton | `config.audit.md` findings 1, 2 |
|
||||||
|
| `pkg/logger` | `Logger`, `errorTracker` globals | `logger.audit.md` finding 1 |
|
||||||
|
| `pkg/modelregistry` | `defaultRegistry` read by 6 functions without the lock | `modelregistry.audit.md` findings 2, 8 *(fixed 2026-09-30)* |
|
||||||
|
| `pkg/tracing` | `tracer` global | `tracing.audit.md` finding 5 *(fixed 2026-09-30)* |
|
||||||
|
| `pkg/errortracking` | `sentry.Init` mutates process globals | `errortracking.audit.md` finding 2 |
|
||||||
|
|
||||||
|
**Failure scenario.** `pkg/config` finding 1 is the sharpest illustration. A
|
||||||
|
concurrent `Manager.Set`/`Manager.Get` pair reaches viper's internal maps, which
|
||||||
|
have no mutex. A concurrent map read and write in Go is not a panic — it is
|
||||||
|
`fatal error: concurrent map read and map write`, which **`recover()` cannot
|
||||||
|
catch**. The process dies instantly, mid-request, with no graceful shutdown and
|
||||||
|
no error-tracker report. That is a remotely-triggerable hard crash, and it
|
||||||
|
cannot be found by inspection at scale — it is precisely what `-race` exists to
|
||||||
|
find. The detector has been in Go since 1.1 and costs one flag.
|
||||||
|
|
||||||
|
**Recommendation.** Add a race job that covers everything, and keep it separate
|
||||||
|
from the coverage run (race builds are ~2–10× slower):
|
||||||
|
|
||||||
|
```makefile
|
||||||
|
test-race:
|
||||||
|
@go test -race -count=1 ./pkg/...
|
||||||
|
```
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
race-tests:
|
||||||
|
name: Race Detector
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
steps:
|
||||||
|
- uses: actions/checkout@v6
|
||||||
|
- uses: actions/setup-go@v6
|
||||||
|
with: { go-version: "1.24" }
|
||||||
|
- run: go test -race -count=1 ./pkg/...
|
||||||
|
```
|
||||||
|
|
||||||
|
Expect it to fail on the first run — that is the point. Fix `pkg/logger`,
|
||||||
|
`pkg/config` and `pkg/cache` first, since they are the shared dependencies. Note
|
||||||
|
that the race detector only reports races that **actually execute**, so X1 and X2
|
||||||
|
have to be fixed together: a race detector pointed at packages with no tests
|
||||||
|
finds nothing.
|
||||||
|
|
||||||
|
**Status (2026-09-30) — resolved for the packages with tests.** `make test-race` now exists
|
||||||
|
(`go test -race -count=1 ./pkg/...`), `test-unit` covers `./pkg/...`, and `test`
|
||||||
|
depends on both. The CI workflow (`.github/workflows/tests.yml`) now has a `race-tests`
|
||||||
|
job running the same command. The first full run was not clean:
|
||||||
|
|
||||||
|
| Package | Race | Kind |
|
||||||
|
|---|---|---|
|
||||||
|
| `pkg/logger` | `Logger` / `errorTracker` reassigned while other goroutines log (hit via `pkg/server` tests) | **production** — now guarded by an `RWMutex` (`getLogger`, `setLogger`, `getErrorTracker`); the exported `Logger` var is kept for compatibility |
|
||||||
|
| `pkg/security` | `DatabaseAuthenticator.Authenticate` passed `&userCtx` to the async session-activity goroutine while also returning it to the caller | **production** — the goroutine now gets a copy and is tracked by a `WaitGroup` so tests can wait for it |
|
||||||
|
| `pkg/security` tests | async activity update used sqlmock concurrently with the test adding expectations | test — tests wait via `authenticateSync` |
|
||||||
|
| `pkg/eventbroker`, `pkg/websocketspec` tests | handler/hook closures mutated a plain `bool`/`int` from worker goroutines | test — now `atomic` |
|
||||||
|
| `pkg/mqttspec` tests | not a race: hand-built `HookContext` lacked `TableName`/`Model`/`ModelPtr`, the unsubscribe test set `Data` instead of `SubscriptionID`, and `:memory:` SQLite gave each pooled connection its own empty database | test — fixed; these were failing without `-race` too |
|
||||||
|
|
||||||
|
`pkg/cache`, `pkg/config`, `pkg/modelregistry`, `pkg/tracing` and
|
||||||
|
`pkg/errortracking` are listed above but did **not** trip the detector: their
|
||||||
|
racing paths are not exercised by the current tests, which is the point made in
|
||||||
|
the paragraph above about X1 and X2 needing to be fixed together. Adding
|
||||||
|
concurrent tests for those globals is still outstanding.
|
||||||
|
|
||||||
|
Known limitation: `pkg/security` tests are not repeatable with `-count>1` (a
|
||||||
|
package-level capability cache carries over between runs), so the race target
|
||||||
|
keeps `-count=1`.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### X2. High — `go test` runs against 2 of 23 packages
|
||||||
|
|
||||||
|
Every `go test` invocation in the repository names exactly
|
||||||
|
`./pkg/resolvespec ./pkg/restheadspec`. No invocation uses `./...` or
|
||||||
|
`./pkg/...`.
|
||||||
|
|
||||||
|
The test bodies that exist but are never executed by CI:
|
||||||
|
|
||||||
|
| Package | Test files | Test lines | Run by CI? |
|
||||||
|
|---|---|---|---|
|
||||||
|
| `restheadspec` | 19 | 5 123 | **yes** |
|
||||||
|
| `resolvespec` | 8 | 2 379 | **yes** |
|
||||||
|
| `security` | 15 | 6 359 | no |
|
||||||
|
| `common` | 10 | 3 644 | no |
|
||||||
|
| `reflection` | 8 | 3 404 | no |
|
||||||
|
| `websocketspec` | 6 | 3 092 | no |
|
||||||
|
| `funcspec` | 3 | 2 416 | no |
|
||||||
|
| `spectypes` | 7 | 2 367 | no |
|
||||||
|
| `eventbroker` | 4 | 1 527 | no |
|
||||||
|
| `mqttspec` | 3 | 1 408 | no |
|
||||||
|
| `middleware` | 5 | 1 127 | no |
|
||||||
|
| `openapi` | 2 | 1 022 | no |
|
||||||
|
| `server` | 2 | 694 | no |
|
||||||
|
| `dbmanager` | 2 | 659 | no |
|
||||||
|
| `config` | 1 | 608 | no |
|
||||||
|
| `cache` | 1 | 69 | no |
|
||||||
|
| `errortracking` | 1 | 67 | no |
|
||||||
|
| `metrics` | 1 | 64 | no |
|
||||||
|
| `resolvemcp` | 1 | 34 | no |
|
||||||
|
| `logger` | 0 | 0 | — |
|
||||||
|
| `modelregistry` | 1 | ~150 | yes (`-race`) *(added 2026-09-30)* |
|
||||||
|
| `testmodels` | 0 | 0 | — |
|
||||||
|
| `tracing` | 1 | ~90 | yes *(added 2026-09-30)* |
|
||||||
|
|
||||||
|
**Failure scenario.** `pkg/security` has 6 359 lines of tests — the largest test
|
||||||
|
body in the repository — and **not one of them runs in CI**. A change that breaks
|
||||||
|
authentication, column-level security or row-security templates merges green.
|
||||||
|
The `maint.yml` job named "Run Vet Tests" is misleading: it runs `go mod
|
||||||
|
download`, `go mod verify` and `go vet ./...` and contains **no `go test` step at
|
||||||
|
all** (verified by grep). So the only signal on 21 of 23 packages is "it
|
||||||
|
compiles and vet is happy".
|
||||||
|
|
||||||
|
This directly explains the density of findings in this audit. The
|
||||||
|
`pkg/modelregistry` authorization fail-open (`modelregistry.audit.md` finding 1)
|
||||||
|
and the `pkg/cache`/`pkg/security` auth-outage-on-cache-failure
|
||||||
|
(`cache.audit.md` finding 1) are both the kind of defect a single unit test would
|
||||||
|
have caught, in packages that have never been tested.
|
||||||
|
|
||||||
|
**Recommendation.** Change every invocation to `./pkg/...`:
|
||||||
|
|
||||||
|
```makefile
|
||||||
|
test-unit:
|
||||||
|
@go test ./pkg/... -v -cover
|
||||||
|
```
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
- name: Run unit tests
|
||||||
|
run: go test ./pkg/... -v -cover
|
||||||
|
```
|
||||||
|
|
||||||
|
If some currently-unrun package fails immediately, that is a bug report, not a
|
||||||
|
reason to keep the narrow list. Quarantine individual failing tests with
|
||||||
|
`t.Skip` and a `TODO` referencing an issue, so the *package* stays in the set.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### X3. High — integration failures cannot fail CI
|
||||||
|
|
||||||
|
`.github/workflows/tests.yml`, `integration-tests` job — every meaningful step
|
||||||
|
carries `continue-on-error: true`:
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
- name: Run resolvespec integration tests
|
||||||
|
continue-on-error: true
|
||||||
|
env:
|
||||||
|
TEST_DATABASE_URL: "host=localhost user=postgres password=postgres dbname=resolvespec_test port=5432 sslmode=disable"
|
||||||
|
run: go test -tags=integration ./pkg/resolvespec -v -coverprofile=coverage-resolvespec-integration.out
|
||||||
|
- name: Run restheadspec integration tests
|
||||||
|
continue-on-error: true
|
||||||
|
...
|
||||||
|
```
|
||||||
|
|
||||||
|
**Failure scenario.** The integration suites are the only tests that exercise
|
||||||
|
real SQL generation against a real PostgreSQL — i.e. the only automated check on
|
||||||
|
the identifier-quoting and filter-construction paths that this audit's threat
|
||||||
|
model cares most about. Because both steps are `continue-on-error`, a SQL
|
||||||
|
injection regression, a broken join, or a total suite failure (wrong DSN, missing
|
||||||
|
migration) shows as a green check mark with a collapsed red step that nobody
|
||||||
|
opens. The job has no step that fails, so the job always passes. This is
|
||||||
|
strictly worse than not having the tests, because it creates the appearance of
|
||||||
|
coverage.
|
||||||
|
|
||||||
|
Note the integration DSN itself uses `sslmode=disable`, consistent with X6.
|
||||||
|
|
||||||
|
**Recommendation.** Remove `continue-on-error` from the two `go test` steps.
|
||||||
|
Keep it only on the coverage-report generation and artifact-upload steps, which
|
||||||
|
genuinely should not fail a build. If the suites are currently flaky, fix or
|
||||||
|
skip the flaky tests individually — `continue-on-error` on the whole step
|
||||||
|
disables the signal entirely.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### X4. Medium — `gosec` is not enabled — **RESOLVED**
|
||||||
|
|
||||||
|
> **Status (2026-09-30):** `gosec` is now in `linters.enable` and the repository lints clean
|
||||||
|
> (0 issues). The initial run produced 115 findings. Real fixes: login-form values in
|
||||||
|
> `security/oauth_server.go` are now HTML-escaped (G705), and `SqlSparseVector` index
|
||||||
|
> parsing uses `ParseInt(..., 10, 32)` (G109). The remaining ~110 sites carry
|
||||||
|
> `//nolint:gosec // Gxxx: <reason>` comments. The G201/G701 reasons (identifiers from
|
||||||
|
> trusted config or internal/validated names) and the G115 range claims were not
|
||||||
|
> individually audited and still need review. The text below describes the state before the change.
|
||||||
|
|
||||||
|
`.golangci.json` (`version: 2`) enables exactly three linters beyond the v2
|
||||||
|
standard set:
|
||||||
|
|
||||||
|
```json
|
||||||
|
"linters": {
|
||||||
|
"enable": [
|
||||||
|
"gocritic",
|
||||||
|
"misspell",
|
||||||
|
"revive"
|
||||||
|
],
|
||||||
|
```
|
||||||
|
|
||||||
|
golangci-lint v2's standard set (`errcheck`, `govet`, `ineffassign`,
|
||||||
|
`staticcheck`, `unused`) is on by default, so those do run. **`gosec` does not** —
|
||||||
|
it appears in the file only inside an exclusion rule for `_test.go`:
|
||||||
|
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"linters": [
|
||||||
|
"dupl",
|
||||||
|
"errcheck",
|
||||||
|
"gocritic",
|
||||||
|
"gosec"
|
||||||
|
],
|
||||||
|
"path": "_test\\.go"
|
||||||
|
},
|
||||||
|
```
|
||||||
|
|
||||||
|
Listing a linter in `exclusions.rules` does not enable it. The `lint` job in
|
||||||
|
`.github/workflows/maint.yml:40-57` does run golangci-lint over the whole
|
||||||
|
repository with `version: latest`, so the config is applied — it simply never
|
||||||
|
asks for the security checks.
|
||||||
|
|
||||||
|
**Failure scenario.** This repository builds SQL by string construction from
|
||||||
|
attacker-controlled schema, table, column and filter names (see
|
||||||
|
`restheadspec.audit.md` and `common.audit.md`). `gosec`'s `G201`/`G202`
|
||||||
|
(SQL string formatting/concatenation) are exactly the rules that would flag a
|
||||||
|
new `fmt.Sprintf` into a query, which is the single most likely way a SQL
|
||||||
|
injection enters this codebase. Also unenabled and relevant: `G104` (unhandled
|
||||||
|
errors — this audit found ~20 discarded errors in `pkg/cache` alone), `G304`
|
||||||
|
(file path from variable — relevant to `PathsConfig.Join`, `config.audit.md`
|
||||||
|
finding 14), `G402` (bad TLS settings — X6), `G404` (weak random).
|
||||||
|
|
||||||
|
**Recommendation.** Add `gosec` to `linters.enable` and triage the initial
|
||||||
|
findings. Expect noise on the SQL rules given the architecture; suppress
|
||||||
|
individual verified-safe sites with `//nolint:gosec // G201: identifier is
|
||||||
|
validated by X` comments that name the invariant, rather than disabling the rule
|
||||||
|
globally. That converts each suppression into a reviewable claim.
|
||||||
|
|
||||||
|
Consider also `bodyclose`, `rowserrcheck` and `sqlclosecheck` for a
|
||||||
|
database-heavy codebase, and `contextcheck` given how many methods here accept a
|
||||||
|
`ctx` and ignore it.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### X5. Medium — unsynchronized mutable package globals are the dominant pattern
|
||||||
|
|
||||||
|
Nine of the twenty-three packages expose mutable process-wide state through
|
||||||
|
package-level variables, and most guard it with nothing:
|
||||||
|
|
||||||
|
| Package | Global | Guarded? |
|
||||||
|
|---|---|---|
|
||||||
|
| `pkg/logger` | `Logger *zap.SugaredLogger` (`logger.go:15`), `errorTracker` (`:16`) | **no** — and `Logger` is exported |
|
||||||
|
| `pkg/cache` | `defaultCache *Cache` (`cache.go:10`) | **no** |
|
||||||
|
| `pkg/config` | `configInstance *Manager` (`manager.go:15`) | **no** |
|
||||||
|
| `pkg/tracing` | `tracer` | **yes** *(fixed 2026-09-30)* — `atomic.Pointer` |
|
||||||
|
| `pkg/modelregistry` | `defaultRegistry` | **yes** *(fixed 2026-09-30)* — guarded by `registriesMutex`; all access via `GetDefaultRegistry()` |
|
||||||
|
| `pkg/metrics` | `globalProvider` (`interfaces.go:50-51`) | **yes** — `globalProviderMu sync.RWMutex` |
|
||||||
|
|
||||||
|
`pkg/metrics` is the model the others should follow:
|
||||||
|
|
||||||
|
```go
|
||||||
|
// pkg/metrics/interfaces.go:50-72
|
||||||
|
var (
|
||||||
|
globalProviderMu sync.RWMutex
|
||||||
|
globalProvider Provider
|
||||||
|
)
|
||||||
|
|
||||||
|
func SetProvider(p Provider) {
|
||||||
|
globalProviderMu.Lock()
|
||||||
|
globalProvider = p
|
||||||
|
globalProviderMu.Unlock()
|
||||||
|
}
|
||||||
|
|
||||||
|
func GetProvider() Provider {
|
||||||
|
globalProviderMu.RLock()
|
||||||
|
p := globalProvider
|
||||||
|
globalProviderMu.RUnlock()
|
||||||
|
if p == nil {
|
||||||
|
return &NoOpProvider{}
|
||||||
|
}
|
||||||
|
return p
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
Note that it also returns a working `NoOpProvider` rather than `nil`, so callers
|
||||||
|
need no nil check — the pattern `pkg/logger` and `pkg/cache` should copy.
|
||||||
|
|
||||||
|
**Failure scenario.** Beyond the data races in X1, the shared failure mode is
|
||||||
|
**lazy initialization on the request path**. `cache.GetDefaultCache()`
|
||||||
|
(`cache.go:48`) and `config.GetConfigManager()` (`manager.go:18`) both
|
||||||
|
`if x == nil { x = construct() }` with no `sync.Once`. Under concurrent first
|
||||||
|
traffic, several instances are constructed and all but one are silently
|
||||||
|
discarded, so writes go to an orphaned object — a cache that is permanently 100%
|
||||||
|
miss, or two `Manager`s disagreeing about configuration. It presents as "the
|
||||||
|
cache doesn't work" with no error anywhere.
|
||||||
|
|
||||||
|
`pkg/logger.Logger` being **exported** and mutable is its own hazard: any
|
||||||
|
package, or any consumer of this library, can reassign the process logger
|
||||||
|
mid-flight while other goroutines are calling `Logger.Infow`.
|
||||||
|
|
||||||
|
**Recommendation.** For each global: `atomic.Pointer[T]` for
|
||||||
|
single-pointer swaps, `sync.Once` for lazy defaults, `sync.RWMutex` for
|
||||||
|
multi-field state. Unexport `logger.Logger` behind accessors. Where a nil global
|
||||||
|
is possible, return a no-op implementation instead of `nil`, as
|
||||||
|
`pkg/metrics.GetProvider` does.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### X6. Medium — insecure transport is the default everywhere
|
||||||
|
|
||||||
|
Every network dependency defaults to cleartext, and in two cases there is no way
|
||||||
|
to configure otherwise:
|
||||||
|
|
||||||
|
| Component | Default | Configurable? | Reference |
|
||||||
|
|---|---|---|---|
|
||||||
|
| PostgreSQL | `sslmode: disable` (`config/manager.go:242`) | yes, via config | `config.audit.md` finding 3 |
|
||||||
|
| OTLP traces | `otlptracegrpc.WithInsecure()` hardcoded (`tracing/tracing.go:41`) *(fixed 2026-09-30: TLS default, `Insecure` opt-in)* | **yes** | `tracing.audit.md` finding 1 |
|
||||||
|
| Redis (cache) | no `TLSConfig` set | **no** — `RedisConfig` has no TLS field | `cache.audit.md` finding 15 |
|
||||||
|
| Memcache | no TLS | **no** | `cache.audit.md` finding 15 |
|
||||||
|
| CORS | `allowed_origins: ["*"]`, `allowed_headers: ["*"]` (`config/manager.go:214-216`) | yes | `config.audit.md` finding 3 |
|
||||||
|
| DB user | `user: postgres` with blank password (`config/manager.go:239-240`) | yes | `config.audit.md` finding 3 |
|
||||||
|
|
||||||
|
The `tracing.go:41` case is the most pointed, because the code knows better:
|
||||||
|
|
||||||
|
```go
|
||||||
|
otlptracegrpc.WithInsecure(), // Use WithTLSCredentials in production
|
||||||
|
```
|
||||||
|
|
||||||
|
The comment names the fix and the config struct provides no way to apply it.
|
||||||
|
|
||||||
|
**Failure scenario.** The cache holds `UserContext` — identity and authorization
|
||||||
|
data — keyed by the raw bearer token (`security/providers.go:398`). With no TLS,
|
||||||
|
anything on the path between the service and Redis can read session contents and
|
||||||
|
the `AUTH` password, then **write** a forged `auth:session:<token>` entry.
|
||||||
|
`GetOrSet` returns a cache hit without consulting the database, so a forged entry
|
||||||
|
is a complete authentication bypass. Meanwhile the trace exporter ships full
|
||||||
|
request URLs including query strings (`tracing.audit.md` finding 2) in cleartext
|
||||||
|
to the collector.
|
||||||
|
|
||||||
|
**Recommendation.** Invert every default: TLS on unless explicitly disabled.
|
||||||
|
Concretely — add `TLS`/`TLSSkipVerify`/`TLSCACertFile` to `cache.RedisConfig`
|
||||||
|
and `tracing.Config`; change the `sslmode` default to `require`; change
|
||||||
|
`cors.allowed_origins` to `[]` and require an explicit list; remove the default
|
||||||
|
`postgres`/blank-password credentials so a misconfigured deployment fails to
|
||||||
|
start rather than connecting to a local database as a superuser. Add a startup
|
||||||
|
validation pass that logs a prominent warning for each insecure setting actually
|
||||||
|
in effect.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### X7. Medium — panic handling is inconsistent, and where it exists it fails open
|
||||||
|
|
||||||
|
Three different conventions coexist:
|
||||||
|
|
||||||
|
1. **`logger.CatchPanic(location)`** (`logger/logger.go:184`) — recovers, logs,
|
||||||
|
reports, and **swallows**. Both call sites are security enforcement:
|
||||||
|
`security/provider.go:302` (`ApplyColumnSecurity`) and `:443`
|
||||||
|
(`GetRowSecurityTemplate`). See `logger.audit.md` finding 4.
|
||||||
|
2. **`logger.HandlePanic(method, r)`** (`logger/logger.go:197`) — converts the
|
||||||
|
panic to an `error` the caller must handle. This is the correct shape.
|
||||||
|
3. **Nothing at all.** `pkg/cache` has zero `recover()` calls in 1 538 lines;
|
||||||
|
so do several other packages.
|
||||||
|
|
||||||
|
**Failure scenario (fail-open).** `ApplyColumnSecurity` panics — a nil map, a
|
||||||
|
bad type assertion on a rule, a reflection edge case. `CatchPanic` recovers and
|
||||||
|
the function returns normally, so the caller believes column security was
|
||||||
|
applied. It was not. The response contains the columns the security layer was
|
||||||
|
supposed to strip. The panic is logged, but the request succeeds with elevated
|
||||||
|
data exposure. A security control whose failure mode is "allow" is the wrong
|
||||||
|
default; it must be "deny".
|
||||||
|
|
||||||
|
**Failure scenario (panic under a lock).** `pkg/cache` holds `m.mu` across
|
||||||
|
`m.items[key] = ...` (`provider_memory.go:111`). After `Close()` sets
|
||||||
|
`items = nil` that assignment panics. With no recover in the package the panic
|
||||||
|
propagates to whatever handler exists upstream; if that handler recovers, `m.mu`
|
||||||
|
is **never unlocked** and every subsequent cache operation blocks forever. The
|
||||||
|
process stays alive and wedged — worse than a crash, because health checks that
|
||||||
|
do not touch the cache keep passing.
|
||||||
|
|
||||||
|
**Recommendation.** Establish one convention and apply it:
|
||||||
|
|
||||||
|
- **Request boundaries** (HTTP handlers, event consumers, goroutines): recover,
|
||||||
|
log with stack, report to the error tracker, return 500 / nack. A `go`
|
||||||
|
statement without a deferred recover is a process-kill waiting to happen —
|
||||||
|
`security/providers.go:447` (`go a.updateSessionActivity(...)`) is one.
|
||||||
|
- **Security enforcement**: recover, log, and **fail closed** — return an error
|
||||||
|
that the caller must propagate as a denial. Never `CatchPanic`.
|
||||||
|
- **Internal helpers**: do not recover. Let the boundary handle it.
|
||||||
|
- **Anything holding a lock**: prefer `defer mu.Unlock()` (already the pattern in
|
||||||
|
`pkg/cache`) so a panic cannot leak the lock, and keep panicking code out of
|
||||||
|
critical sections.
|
||||||
|
|
||||||
|
Add a `CatchPanicFailClosed(location string, err *error)` helper so the
|
||||||
|
fail-closed variant is as easy to reach for as `CatchPanic`.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### X8. Medium — attacker data reaches Sentry unscrubbed
|
||||||
|
|
||||||
|
Two facts compose badly:
|
||||||
|
|
||||||
|
`pkg/logger/logger.go:125-140` — every `Error` (and every `Warn`, `:108-123`)
|
||||||
|
forwards the fully-formatted message to the error tracker:
|
||||||
|
|
||||||
|
```go
|
||||||
|
func Error(template string, args ...interface{}) {
|
||||||
|
ctx, remainingArgs := extractContext(args...)
|
||||||
|
message := fmt.Sprintf(template, remainingArgs...)
|
||||||
|
...
|
||||||
|
if errorTracker != nil {
|
||||||
|
errorTracker.CaptureMessage(ctx, message, errortracking.SeverityError, map[string]interface{}{
|
||||||
|
"process_id": os.Getpid(),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
And `pkg/errortracking` installs **no `BeforeSend` scrubber**
|
||||||
|
(`errortracking.audit.md` finding 1), so the message goes to Sentry verbatim.
|
||||||
|
|
||||||
|
Meanwhile error strings across the codebase interpolate attacker-controlled
|
||||||
|
values, sometimes secrets:
|
||||||
|
|
||||||
|
| Site | Interpolated value |
|
||||||
|
|---|---|
|
||||||
|
| `cache/cache_manager.go:26`, `:40` | the full cache key — for the session cache, **the raw bearer token** |
|
||||||
|
| `security/providers.go:391` | the raw `Authorization` header, logged at `Warn` when multiple tokens are present |
|
||||||
|
| `config/manager.go:164` | config file paths |
|
||||||
|
| throughout `restheadspec` | schema, table, column and filter values from the request |
|
||||||
|
|
||||||
|
**Failure scenario.** `security/providers.go:391` is live today:
|
||||||
|
|
||||||
|
```go
|
||||||
|
logger.Warn("Multiple authentication tokens provided in Authorization header (%d tokens). This is unusual and may indicate a misconfigured client. Header: %s", len(tokens), sessionToken)
|
||||||
|
```
|
||||||
|
|
||||||
|
A client sends two bearer tokens. `logger.Warn` formats the full header value
|
||||||
|
into the message and forwards it to Sentry, where a **valid session credential**
|
||||||
|
is now stored by a third party, visible to everyone with Sentry access, retained
|
||||||
|
per Sentry's policy, and replayable for the token's lifetime. No attacker
|
||||||
|
sophistication is required — the trigger is a single extra header, and the
|
||||||
|
codebase invites it by logging the header contents as the diagnostic.
|
||||||
|
|
||||||
|
**Recommendation.**
|
||||||
|
|
||||||
|
1. Add a `BeforeSend` hook in `pkg/errortracking` that redacts
|
||||||
|
`Authorization`, `Cookie`, `Set-Cookie`, anything matching
|
||||||
|
`(?i)(token|password|secret|apikey|api_key|bearer)\s*[:=]\s*\S+`, and
|
||||||
|
long high-entropy strings. This is the one change that bounds the whole class.
|
||||||
|
2. Never log a credential, even truncated. Change `providers.go:391` to log
|
||||||
|
`len(tokens)` only.
|
||||||
|
3. Replace `fmt.Errorf("key not found: %s", key)` with a sentinel
|
||||||
|
`cache.ErrNotFound` (`cache.audit.md` finding 7).
|
||||||
|
4. Key the session cache on `sha256(token)`, as
|
||||||
|
`security/keystore_database.go:287` already does for API keys.
|
||||||
|
5. Add sampling / rate limiting to the tracker fan-out
|
||||||
|
(`logger.audit.md` finding 3) so an error storm is not also a cost and
|
||||||
|
availability event.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### X9. Low — five packages have no tests at all
|
||||||
|
|
||||||
|
`pkg/logger`, `pkg/modelregistry`, `pkg/testmodels`, `pkg/tracing` have zero
|
||||||
|
`*_test.go` files. `pkg/resolvemcp` has 34 lines, `pkg/metrics` 64,
|
||||||
|
`pkg/errortracking` 67, `pkg/cache` 69.
|
||||||
|
|
||||||
|
**Failure scenario.** `pkg/modelregistry` is untested and contains this audit's
|
||||||
|
only **Critical** authorization finding: `GetModel` returns a "registry locked"
|
||||||
|
error under write-lock contention, which `security/hooks.go:274-294` converts
|
||||||
|
into `return nil // model not registered, allow by default`
|
||||||
|
(`modelregistry.audit.md` finding 1; *fixed 2026-09-30, regression tests added*). A twenty-line test that registers a model
|
||||||
|
from one goroutine while reading it from another would demonstrate the fail-open
|
||||||
|
immediately. The package guards a security boundary and has never been tested.
|
||||||
|
|
||||||
|
`pkg/logger` being untested matters for a different reason: it is imported by
|
||||||
|
almost every other package, so a defect there (the format-string sink in `Info`
|
||||||
|
and `Debug`, `logger.audit.md` finding 6) is repo-wide.
|
||||||
|
|
||||||
|
**Recommendation.** Prioritize by blast radius, not by size:
|
||||||
|
|
||||||
|
1. `pkg/modelregistry` — concurrent register/read; assert `GetModelRulesByName`
|
||||||
|
never returns a "locked" error that a caller could read as "not registered".
|
||||||
|
2. `pkg/logger` — nil-`Logger` fallback paths, format-string handling, and that
|
||||||
|
`Warn`/`Error` do not forward secrets once a scrubber exists.
|
||||||
|
3. `pkg/cache` — concurrent `GetDefaultCache`, the expired-item TOCTOU, and that
|
||||||
|
`tagToKeys` does not grow after eviction.
|
||||||
|
4. `pkg/tracing`, `pkg/metrics`, `pkg/errortracking` — construction and no-op
|
||||||
|
paths; these are mostly configuration surfaces.
|
||||||
|
|
||||||
|
Combine with X1 and X2: tests that are not run, and tests run without `-race`,
|
||||||
|
do not close these gaps.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### X10. High — configured subsystems that are never installed
|
||||||
|
|
||||||
|
Three separate subsystems are fully built — typed config, defaults, tests,
|
||||||
|
documentation — and then never connected to anything that runs.
|
||||||
|
|
||||||
|
**1. Every protective middleware.** `pkg/middleware` provides rate limiting, IP
|
||||||
|
blacklisting, request-size limiting and input sanitization. Non-test callers:
|
||||||
|
|
||||||
|
| Constructor | Non-test callers |
|
||||||
|
|---|---|
|
||||||
|
| `middleware.NewRateLimiter` | **0** |
|
||||||
|
| `middleware.NewIPBlacklist` | **0** |
|
||||||
|
| `middleware.NewRequestSizeLimiter` | **0** |
|
||||||
|
| `middleware.DefaultSanitizer` | **0** outside the package |
|
||||||
|
| `middleware.StrictSanitizer` | **0** |
|
||||||
|
| `middleware.PanicRecovery` | 1 — `pkg/server/manager.go:466` |
|
||||||
|
|
||||||
|
`pkg/server/manager.go` is the only file outside the package that imports it, and
|
||||||
|
only for `PanicRecovery`. The config that exists to drive the rest —
|
||||||
|
`MiddlewareConfig.RateLimitRPS`, `.RateLimitBurst`, `.MaxRequestSize`
|
||||||
|
(`pkg/config/config.go:123-125`), defaulted at `pkg/config/manager.go:209-211` —
|
||||||
|
has **no reader anywhere in the module**.
|
||||||
|
|
||||||
|
**2. The metrics provider.** `metrics.SetProvider` and
|
||||||
|
`metrics.NewPrometheusProvider` have **0 non-test callers**, so
|
||||||
|
`metrics.GetProvider()` returns `&NoOpProvider{}`
|
||||||
|
(`pkg/metrics/interfaces.go:63-72`) for the process lifetime. Every instrumented
|
||||||
|
call site in the repository — 39 DB-query sites in
|
||||||
|
`pkg/common/adapters/database`, the HTTP middleware, the event-broker counters,
|
||||||
|
and the sole `RecordPanic` call at `pkg/middleware/panic.go:19` — writes to a
|
||||||
|
no-op. `MetricsConfig.Enabled` and `.Provider` are likewise never read, and
|
||||||
|
`pkg/config` has no `metrics` section at all.
|
||||||
|
|
||||||
|
**3. The configured CORS policy.** `config.CORSConfig`
|
||||||
|
(`pkg/config/config.go:128-134`), defaulted at `pkg/config/manager.go:214-217`,
|
||||||
|
is never read. The policy that actually applies comes from a **different type of
|
||||||
|
the same name**, `common.CORSConfig`, built by `common.DefaultCORSConfig()`
|
||||||
|
(`pkg/common/cors.go:19-48`), which derives allowed origins from the configured
|
||||||
|
server instances and the host's local IPs and ignores `cors.allowed_origins`
|
||||||
|
entirely. It is called from ten sites across `pkg/resolvespec` and
|
||||||
|
`pkg/restheadspec`.
|
||||||
|
|
||||||
|
**Failure scenario.** Each of these is a silent, config-shaped lie, and they fail
|
||||||
|
in the same way: the operator's mental model of the deployment is wrong in the
|
||||||
|
direction of believing a control exists.
|
||||||
|
|
||||||
|
- **Under the hostile-client threat model there is no rate limit and no
|
||||||
|
request-body limit in the serving path.** `max_request_size: 10485760` is
|
||||||
|
configured and unenforced, so a single unauthenticated `POST` with a
|
||||||
|
multi-gigabyte body is read into memory and OOM-kills the process; unlimited
|
||||||
|
request rate exhausts the 25-connection default pool
|
||||||
|
(`pkg/config/manager.go:224`) just as cheaply. Both are one-line attacks
|
||||||
|
against controls the configuration says are active. An operator lowering
|
||||||
|
`rate_limit_rps` during an incident observes no change and will reasonably
|
||||||
|
conclude the attack exceeds the limit rather than that no limit exists.
|
||||||
|
- **There is no telemetry with which to notice any of it.** No request counts, no
|
||||||
|
latency histograms, no `panics_total`, no DB-query metrics — the one signal
|
||||||
|
that would show an attack in progress is wired end to end and discarded at the
|
||||||
|
last step. This is also why the metrics cardinality defects
|
||||||
|
(`metrics.audit.md` findings 2 and 5) are only latent: they become live the
|
||||||
|
moment someone installs the provider that the config implies is already there.
|
||||||
|
- **Tightening `cors.allowed_origins` does nothing.** The value is ignored, so a
|
||||||
|
hardening change lands, reviews clean, deploys, and changes no behaviour. Two
|
||||||
|
types named `CORSConfig` in two packages is the mechanism; nothing warns.
|
||||||
|
|
||||||
|
The common thread is that none of this fails visibly. It compiles, the tests pass
|
||||||
|
(`pkg/middleware` has the repo's best test ratio — 1 127 test lines to 799 code
|
||||||
|
lines — all of it exercising code nothing calls), CI is green, and the config file
|
||||||
|
documents features that are absent. Under X2 these packages are not even in the
|
||||||
|
tested set, so the tests that do exist are not run.
|
||||||
|
|
||||||
|
**Recommendation.**
|
||||||
|
|
||||||
|
1. **Wire the middleware chain** in `pkg/server` from `MiddlewareConfig`,
|
||||||
|
outermost first: size limiter → rate limiter → blacklist → `PanicRecovery`
|
||||||
|
(innermost, so it sees handler panics; `trackRequestsMiddleware` at
|
||||||
|
`manager.go:540` correctly stays outside). Fix the trusted-proxy handling
|
||||||
|
(`middleware.audit.md` findings 2 and 3) **before** mounting the two IP-based
|
||||||
|
layers, and do not mount the sanitizer at all until findings 5–7 there are
|
||||||
|
resolved — as written it corrupts filter values and can synthesize a
|
||||||
|
`javascript:` URI.
|
||||||
|
2. **Install a metrics provider** from config, gated on `metrics.enabled`, and
|
||||||
|
add the missing `metrics` section to `pkg/config`. Bound the label sets first
|
||||||
|
(`metrics.audit.md` findings 2 and 5) — installing the provider as-is converts
|
||||||
|
two latent cardinality DoS findings into live ones.
|
||||||
|
3. **Delete the duplicate `CORSConfig`** or make `common.DefaultCORSConfig()`
|
||||||
|
read `config.CORSConfig`. Two types with one name, one of them ignored, is a
|
||||||
|
trap regardless of which way it is resolved.
|
||||||
|
4. **Make the class of defect detectable.** Log at startup which middleware,
|
||||||
|
metrics provider and CORS policy are active, so an unwired subsystem is
|
||||||
|
visible in the first ten lines of a boot log instead of during an incident.
|
||||||
|
A CI check that every `mapstructure` field in `pkg/config` has at least one
|
||||||
|
reader would have caught all three of these; so would enabling `unused` in
|
||||||
|
`.golangci.json` for exported-but-unreferenced constructors.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Recommended order of work
|
||||||
|
|
||||||
|
1. **X2 + X1** — point `go test` at `./pkg/...` and add a `-race` job. Everything
|
||||||
|
else in this audit is easier to verify once these exist, and they will
|
||||||
|
surface the six data races on their own.
|
||||||
|
2. **X3** — remove `continue-on-error` from the integration `go test` steps.
|
||||||
|
3. **X10** — mount the request-size limiter and rate limiter. Until this is
|
||||||
|
done the service has no volumetric protection at all, and no metrics with
|
||||||
|
which to see that. Fix `middleware.audit.md` findings 2 and 3 in the same
|
||||||
|
change, since mounting the IP-based layers without them adds attack surface.
|
||||||
|
4. **X8 item 1 and 2** — add the Sentry `BeforeSend` scrubber and stop logging
|
||||||
|
the `Authorization` header. Small, self-contained, stops an active credential
|
||||||
|
leak.
|
||||||
|
5. **X7** — decide the panic convention; make the two `CatchPanic` sites in
|
||||||
|
`pkg/security` fail closed, and stop returning the panic value to the client
|
||||||
|
(`pkg/middleware/panic.go:28`).
|
||||||
|
6. **X5** — fix the globals in `pkg/logger`, `pkg/config`, `pkg/cache`
|
||||||
|
(the shared dependencies) first.
|
||||||
|
7. **X6** — add TLS fields and invert the defaults.
|
||||||
|
8. **X4** — enable `gosec` and triage.
|
||||||
|
9. **X9** — backfill tests, in the order listed above.
|
||||||
|
|
||||||
|
## Per-package audits
|
||||||
|
|
||||||
|
`cache` · `common` · `config` · `dbmanager` · `errortracking` · `eventbroker` ·
|
||||||
|
`funcspec` · `logger` · `metrics` · `middleware` · `modelregistry` · `mqttspec` ·
|
||||||
|
`openapi` · `reflection` · `resolvemcp` · `resolvespec` · `restheadspec` ·
|
||||||
|
`security` · `server` · `spectypes` · `testmodels` · `tracing` · `websocketspec`
|
||||||
|
|
||||||
|
Each is `audit/pkg/<name>.audit.md`.
|
||||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,407 @@
|
|||||||
|
# Audit: `pkg/common`
|
||||||
|
|
||||||
|
| | |
|
||||||
|
|---|---|
|
||||||
|
| **Package** | `github.com/bitechdev/ResolveSpec/pkg/common` (+ `adapters/database`, `adapters/router`) |
|
||||||
|
| **Files** | `sql_helpers.go` (1060), `recursive_crud.go` (645), `validation.go` (444), `json_column.go` (402), `interfaces.go` (311), `spatial_helpers.go` (317), `handler_utils.go` (309), `json_condition.go` (219), `types.go` (192), `cors.go` (156), `handler_example.go` (97); `adapters/database/bun.go` (1767), `pgsql.go` (1600), `gorm.go` (1018), `query_metrics.go` (335), `pgsql_preload_example.go` (275), `pgsql_example.go` (176), `test_helpers.go` (132), `utils.go` (117); `adapters/router/mux.go` (238), `bunrouter.go` (214) |
|
||||||
|
| **Audit date** | 2026-09-30 |
|
||||||
|
| **Axes** | thread locking/waiting, slowness, security, panic handling & logging |
|
||||||
|
| **Threat model** | hostile internet client; request bodies, headers, query params, schema/table/column names all attacker-controlled |
|
||||||
|
| **Depth** | deep for `sql_helpers.go`, `validation.go`, `cors.go`, `recursive_crud.go`, `json_column.go`/`json_condition.go` and the reconnect/transaction paths of the three DB adapters; medium for the rest; the `*_example.go` files were skimmed. Findings 1–3 were verified with throw-away probe tests, which were deleted afterwards |
|
||||||
|
|
||||||
|
## Summary
|
||||||
|
|
||||||
|
`pkg/common` is the shared core behind every spec handler. It contains the
|
||||||
|
`Database` / `SelectQuery` abstraction and its Bun, GORM and raw-`pgx`
|
||||||
|
adapters, the request-option types, column validation, JSON-column parsing,
|
||||||
|
nested (recursive) CRUD, CORS, and a set of SQL string helpers. The spec
|
||||||
|
packages feed **client-supplied raw SQL fragments** through those helpers:
|
||||||
|
`x-custom-sql-w`, `x-custom-sql-or`, `x-custom-sql-join`, preload `where`,
|
||||||
|
sort expressions and cursor filters.
|
||||||
|
|
||||||
|
The central problem is that **`SanitizeWhereClause` / `validateWhereClauseSecurity`
|
||||||
|
is a keyword denylist applied to raw SQL**, and the result is concatenated
|
||||||
|
straight into the query. A denylist can't make arbitrary client SQL safe, and
|
||||||
|
this one misses subqueries, functions, comments and parenthesis balancing.
|
||||||
|
With the helpers exactly as the handlers call them, a client can:
|
||||||
|
|
||||||
|
- escape the outer parentheses and OR past every filter the server adds
|
||||||
|
afterwards (**row security, tenant filters, the PK filter**). This is
|
||||||
|
verified. Row security is inert anyway today (`security.audit.md` finding 2),
|
||||||
|
but this bug will defeat it as soon as that is fixed;
|
||||||
|
- read any table the DB role can see through a subquery;
|
||||||
|
- stall a connection with `pg_sleep`;
|
||||||
|
- bypass the keyword list with a comment (`delete/**/from`).
|
||||||
|
|
||||||
|
Meanwhile, legitimate filters that merely *contain* a word like `update` are
|
||||||
|
silently dropped, and the query runs **unfiltered** (fail-open).
|
||||||
|
|
||||||
|
Other headline findings:
|
||||||
|
|
||||||
|
- `SetCORSHeaders` reflects **any** Origin with `Allow-Credentials: true` and
|
||||||
|
ignores `AllowedOrigins`.
|
||||||
|
- Nested CRUD updates and deletes child rows by primary key alone. A client can
|
||||||
|
modify or delete (or re-parent) any row in a related table.
|
||||||
|
- Sort validation lets arbitrary SQL through whenever a custom join has no
|
||||||
|
alias.
|
||||||
|
|
||||||
|
On the question that started this audit (idle connections becoming unusable),
|
||||||
|
the relevant part of `pkg/common` is the adapters' reconnect logic (finding 5).
|
||||||
|
It is only partly wired into the Bun and pgx adapters, and it's what calls
|
||||||
|
`dbmanager`'s destructive `Reconnect` (`dbmanager.audit.md` findings 1–2).
|
||||||
|
|
||||||
|
## Findings
|
||||||
|
|
||||||
|
| # | Severity | Axis | Finding |
|
||||||
|
|---|---|---|---|
|
||||||
|
| 1 | **Critical** | security | Client raw-SQL WHERE (`x-custom-sql-w`/`-or`, preload where, cursor) is protected only by a keyword denylist: parenthesis escape defeats server-added filters; subqueries, `pg_sleep` and comment bypasses all pass (verified) |
|
||||||
|
| 2 | **Critical** | security | `SetCORSHeaders` reflects any `Origin` and sends `Access-Control-Allow-Credentials: true`; `AllowedOrigins` is never consulted |
|
||||||
|
| 3 | **High** | security | Sort validation: an empty join alias makes `strings.Contains(col, "")` accept **any** sort string, and `(…)` sort expressions allow arbitrary subqueries (verified) |
|
||||||
|
| 4 | **High** | security | Nested CRUD (`recursive_crud.go`) updates and deletes child rows by `WHERE pk = ?` only, with no parent/ownership constraint; a client-supplied `_request` switches the operation per object |
|
||||||
|
| 5 | **High** | locking / availability | Adapter reconnect is inconsistent (Bun and pgx query builders never reconnect) and, where it exists, calls dbmanager's pool-closing `Reconnect`; `BunAdapter.CommitTx`/`RollbackTx` are silent no-ops |
|
||||||
|
| 6 | **Medium** | security / correctness | `SanitizeWhereClause` fails open: on a denylist hit it returns `""`, so the client's filter is dropped and the query returns unfiltered rows; false positives on ordinary data (`'awaiting update'`, `last_update`) |
|
||||||
|
| 7 | **Medium** | security | Any column name starting with `cql` passes `ValidateColumn` unconditionally |
|
||||||
|
| 8 | **Medium** | slowness | Request bodies are read with unbounded `io.ReadAll` in both router adapters |
|
||||||
|
| 9 | **Medium** | logging | Failed queries log the fully interpolated SQL, and nested CRUD logs full row data, at `Error`, which is forwarded to Sentry (`_CROSS-CUTTING.audit.md` X8) |
|
||||||
|
| 10 | **Low** | correctness | `stripEmptyComparisonClauses` regexes rewrite SQL without respecting string literals; quote tracking ignores `''`; `qualifyColumnInCondition` compiles a regex per call |
|
||||||
|
| 11 | **Low** | locking | Adapter fields read without their mutex (`BunAdapter.NewSelect` `db: b.db`, `DriverName`, `PgSQLAdapter.GetUnderlyingDB`) race with `reconnectDB` |
|
||||||
|
| 12 | **Info** | — | `json_column.go` / `json_condition.go` are well built: allowlisted casts, path bound as a single `text[]` parameter, identifiers validated and quoted |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### 1. Critical — Client raw-SQL WHERE is guarded only by a keyword denylist
|
||||||
|
|
||||||
|
`sql_helpers.go:118-161` (`validateWhereClauseSecurity`), `169-305`
|
||||||
|
(`SanitizeWhereClause`), `375-395` (`EnsureOuterParentheses`).
|
||||||
|
|
||||||
|
Call sites that pass **client-controlled** strings:
|
||||||
|
|
||||||
|
| Source | Call site |
|
||||||
|
|---|---|
|
||||||
|
| `x-custom-sql-w` | `restheadspec/handler.go:692-699` → `query.Where(...)` |
|
||||||
|
| `x-custom-sql-or` | `restheadspec/handler.go:703-710` → `query.WhereOr(...)` |
|
||||||
|
| `x-custom-sql-join` | `restheadspec/headers.go:666`, `1330` (sanitized with `tableName ""`) |
|
||||||
|
| preload `where` | `resolvespec/handler.go:2386, 2458`; `restheadspec/handler.go:618, 1170` |
|
||||||
|
| cursor filters | `resolvespec/handler.go:458`; `restheadspec/handler.go:916`; `resolvemcp/handler.go:304` |
|
||||||
|
|
||||||
|
The pipeline is `AddTablePrefixToColumns` → `SanitizeWhereClause` →
|
||||||
|
`EnsureOuterParentheses` → `query.Where(s)`, with **no bind arguments**. The
|
||||||
|
only security check is a substring search for `delete `, `update `, `drop `,
|
||||||
|
`;delete`, and similar.
|
||||||
|
|
||||||
|
A probe reproduced the handler pipeline and then appended a server-side filter
|
||||||
|
`Where("tenant = ?", 5)`, which is what a row-security or tenant hook does:
|
||||||
|
|
||||||
|
| Client `x-custom-sql-w` | Resulting SQL / effect |
|
||||||
|
|---|---|
|
||||||
|
| `1=1)) OR ((1=1` | `WHERE ((1=1)) OR ((1=1)) AND (tenant = 5)`: **every tenant's rows**, because `AND` binds tighter than `OR` |
|
||||||
|
| `id = 1 or (select count(*) from pg_shadow) > 0` | passes unchanged, so boolean-oracle exfiltration from any readable table works |
|
||||||
|
| `id = 1 and pg_sleep(5) is not null` | passes; each request pins a pool connection for as long as the client likes |
|
||||||
|
| `id = 1; delete/**/from items` | passes, because the comment defeats `"delete "` (whether it executes depends on the driver's multi-statement handling) |
|
||||||
|
|
||||||
|
`EnsureOuterParentheses` only checks whether the string *already* starts and
|
||||||
|
ends with a matching pair. It never checks that the parentheses inside are
|
||||||
|
balanced, which is what the escape relies on. `x-custom-sql-or` is worse by
|
||||||
|
design: `WhereOr` ORs the client clause against **every** condition already on
|
||||||
|
the query, so it needs no escape at all to widen a server-side filter.
|
||||||
|
|
||||||
|
Row security currently has no effect (`security.audit.md` finding 2). Fixing
|
||||||
|
that type assertion will **not** give tenant isolation while these headers
|
||||||
|
exist, and the same escape defeats the server's own PK scoping
|
||||||
|
(`restheadspec/handler.go:759-766`).
|
||||||
|
|
||||||
|
**Failure scenario.** An authenticated user of tenant A sends
|
||||||
|
`X-Custom-SQL-W: 1=1)) OR ((1=1` on a list endpoint and receives tenant B's
|
||||||
|
rows. Or they send
|
||||||
|
`X-Custom-SQL-W: (select substr(passwd,1,1) from pg_shadow limit 1) = 'm'` and
|
||||||
|
extract data one character at a time.
|
||||||
|
|
||||||
|
**Recommendation.** Stop accepting raw SQL from clients. Remove the
|
||||||
|
`x-custom-sql-*` headers from the public surface, or gate them behind an
|
||||||
|
explicit server-side allowlist per endpoint. Route client filtering through
|
||||||
|
the structured `FilterOption` path, which validates column names and binds
|
||||||
|
values. If raw fragments have to stay for trusted internal callers:
|
||||||
|
|
||||||
|
- parse them properly (for example with `pg_query_go`) and allow only column
|
||||||
|
references, literals and comparison operators;
|
||||||
|
- reject subqueries and function calls;
|
||||||
|
- verify that parentheses are balanced outside string literals;
|
||||||
|
- apply server-side security predicates last, as a wrapper
|
||||||
|
`WHERE (server) AND (client)`, and never let `WhereOr` attach at top level.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### 2. Critical — CORS reflects every Origin with credentials
|
||||||
|
|
||||||
|
`cors.go:117-155`:
|
||||||
|
|
||||||
|
```go
|
||||||
|
origin := r.Header("Origin")
|
||||||
|
if origin == "" {
|
||||||
|
origin = "*"
|
||||||
|
} else { ... Vary: Origin }
|
||||||
|
w.SetHeader("Access-Control-Allow-Origin", origin)
|
||||||
|
...
|
||||||
|
requestedHeaders := r.Header("Access-Control-Request-Headers")
|
||||||
|
if requestedHeaders != "" {
|
||||||
|
w.SetHeader("Access-Control-Allow-Headers", requestedHeaders)
|
||||||
|
}
|
||||||
|
...
|
||||||
|
if origin != "*" {
|
||||||
|
w.SetHeader("Access-Control-Allow-Credentials", "true")
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
`DefaultCORSConfig` (`cors.go:19-48`) carefully builds `AllowedOrigins` from the
|
||||||
|
server config, and `SetCORSHeaders` **never reads it**. Any site the victim
|
||||||
|
visits can make credentialed cross-origin requests and read the responses.
|
||||||
|
Allowed request headers are also reflected, so `Authorization` and every
|
||||||
|
`X-Custom-SQL-*` header pass preflight. `SetCORSHeaders` is called on every
|
||||||
|
route in `resolvespec/resolvespec.go` (lines 56-347), and `restheadspec` follows
|
||||||
|
the same pattern.
|
||||||
|
|
||||||
|
**Failure scenario.** A user logged in through cookie auth (`SetSessionCookie`/`GetSessionCookie`,
|
||||||
|
`pkg/security/middleware.go:512-540`) visits `evil.example`. Its script calls
|
||||||
|
`fetch("https://api/…/users", {credentials:"include"})` and reads every record
|
||||||
|
the user can see. Combined with finding 1, it can read other tenants' records
|
||||||
|
too.
|
||||||
|
|
||||||
|
**Recommendation.** Send `Allow-Origin: <origin>` and `Allow-Credentials`
|
||||||
|
only when `origin` is in `config.AllowedOrigins`, matched exactly. Otherwise
|
||||||
|
omit the CORS headers. Check requested headers against `AllowedHeaders`
|
||||||
|
instead of echoing them. Build `exposeHeaders` in a fresh slice: `append` onto
|
||||||
|
`config.AllowedHeaders` can write into a shared backing array.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### 3. High — Sort validation bypasses
|
||||||
|
|
||||||
|
`validation.go:271-301`:
|
||||||
|
|
||||||
|
```go
|
||||||
|
foundJoin := false
|
||||||
|
for _, j := range options.JoinAliases {
|
||||||
|
if strings.Contains(sort.Column, j) { // j may be ""
|
||||||
|
```
|
||||||
|
|
||||||
|
`restheadspec/headers.go:674-678` deliberately appends `""` to `JoinAliases`
|
||||||
|
when `extractJoinAlias` can't find an alias (for example
|
||||||
|
`LEFT JOIN t ON …` with no alias, or a LATERAL join without one).
|
||||||
|
`strings.Contains(x, "")` is always `true`, so **any** sort string is
|
||||||
|
accepted. `restheadspec/handler.go:790-793` then passes anything containing a
|
||||||
|
`.` or wrapped in `(…)` to `OrderExpr` **verbatim**. Even with a real alias,
|
||||||
|
the check is a substring test, so a sort like `j.id, (select …)` passes for
|
||||||
|
alias `j`.
|
||||||
|
|
||||||
|
Separately, `(…)` sort expressions are checked by `IsSafeSortExpression`
|
||||||
|
(`validation.go:381-427`), another denylist. It blocks DML keywords, comments
|
||||||
|
and `;`, but allows subqueries and functions.
|
||||||
|
|
||||||
|
Probe results: with `JoinAliases: [""]`, sort `x.id, (select pg_sleep(10))`
|
||||||
|
was kept. With no joins, sort `(select passwd from pg_shadow limit 1)` was kept.
|
||||||
|
|
||||||
|
**Failure scenario.** A client sends a custom join with no alias plus an
|
||||||
|
arbitrary ORDER BY expression, which gives injection in ORDER BY: time-based
|
||||||
|
DoS, or data extraction via `ORDER BY (CASE WHEN (subquery) THEN a ELSE b END)`.
|
||||||
|
|
||||||
|
**Recommendation.** Skip empty aliases. Match `alias + "."` as a prefix, then
|
||||||
|
validate the column after the dot against the joined table. Drop client
|
||||||
|
supplied sort *expressions*, or restrict them to a server-registered set
|
||||||
|
(the `cql` computed columns already provide this).
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### 4. High — Nested CRUD modifies arbitrary related rows
|
||||||
|
|
||||||
|
`recursive_crud.go:64-67, 144-196, 344-380, 395-520`.
|
||||||
|
|
||||||
|
- Children are updated with `UPDATE <related> SET … WHERE pk = ?`, deleted with
|
||||||
|
`DELETE FROM <related> WHERE pk = ?`, and both use the child's PK **from the
|
||||||
|
request body**. Nothing checks that the child belongs to the parent being
|
||||||
|
written or to the caller's tenant.
|
||||||
|
- For updates, the parent's FK is injected into the child data
|
||||||
|
(`recursive_crud.go:495-520`), so updating a foreign child **moves it under
|
||||||
|
the attacker's parent** as well.
|
||||||
|
- `_request` (`recursive_crud.go:64-67, 205-212`) lets the client choose
|
||||||
|
`insert`/`update`/`delete` for each nested object, independent of the HTTP
|
||||||
|
method or the operation the top-level handler authorised.
|
||||||
|
- These statements go straight to `p.db`, so the spec handlers' Before*/After*
|
||||||
|
hooks, and any row-security or audit hooks, don't run for nested rows.
|
||||||
|
|
||||||
|
**Failure scenario.** A client sends a `PUT /orders/1` whose body includes
|
||||||
|
`"lines": [{"id": 9999, "_request": "delete"}]`. Row 9999 of `order_lines` is
|
||||||
|
deleted even if it belongs to another customer's order.
|
||||||
|
|
||||||
|
**Recommendation.** For has-many and has-one children, add
|
||||||
|
`AND <fk> = <parentID>` to update and delete statements, and treat
|
||||||
|
`RowsAffected() == 0` as a forbidden or not-found error. Run the same hook
|
||||||
|
chain (including row security) for nested rows. Allow `_request` only for
|
||||||
|
operations the top-level request is authorised to perform.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### 5. High — Reconnect logic is partial, and it triggers pool destruction
|
||||||
|
|
||||||
|
`adapters/database/bun.go:131-143, 167-186, 226-273, 1298-1318`;
|
||||||
|
`pgsql.go:58-79, 81, 134, 160, 220`; `gorm.go:55, 122-134`.
|
||||||
|
|
||||||
|
- **Coverage is uneven.** `BunAdapter` only retries after reconnecting in
|
||||||
|
`Exec`, `Query`, `BeginTx` and `RunInTransaction`. `NewSelect`, `NewInsert`,
|
||||||
|
`NewUpdate` and `NewDelete` capture `getDB()` once, and `BunSelectQuery.Scan`,
|
||||||
|
`ScanModel`, `Count` and `Exists` call bun directly with no retry. Those are
|
||||||
|
the paths every read handler uses. `PgSQLAdapter` query builders have no
|
||||||
|
reconnect either. Only `GormAdapter` wires `reconnect` into its
|
||||||
|
select, insert, update and delete builders.
|
||||||
|
- **Where it exists, it's harmful.** `reconnectDB` calls the dbmanager factory,
|
||||||
|
which runs `sqlConnection.Reconnect` and closes the pool shared by every
|
||||||
|
other adapter and handle (`dbmanager.audit.md` findings 1–2). Concurrent
|
||||||
|
failures each call the factory.
|
||||||
|
- **Detection is a substring match.** `isDBClosed` (`pgsql.go:72`) matches
|
||||||
|
`"sql: database is closed"`. That only happens *after* someone closed the
|
||||||
|
pool, so the reconnect mechanism mainly exists to recover from damage it
|
||||||
|
causes itself. It does nothing for the real idle-socket failure (a hang, or
|
||||||
|
`driver.ErrBadConn`, which `database/sql` already retries).
|
||||||
|
- `BunAdapter.CommitTx` / `RollbackTx` (`bun.go:239-249`) return `nil` without
|
||||||
|
doing anything. A caller using the `BeginTx`-less path gets
|
||||||
|
"committed" when nothing happened. `BunTxAdapter` is correct.
|
||||||
|
|
||||||
|
**Recommendation.** Remove adapter-level reconnect entirely and rely on
|
||||||
|
`database/sql`'s pool (see the fix order in `dbmanager.audit.md`). Make
|
||||||
|
`BunAdapter.CommitTx`/`RollbackTx` return an explicit
|
||||||
|
"not in a transaction" error. Add a per-query `context.WithTimeout` in the
|
||||||
|
adapters as the single place where query deadlines are enforced.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### 6. Medium — `SanitizeWhereClause` fails open and has false positives
|
||||||
|
|
||||||
|
`sql_helpers.go:176-179`:
|
||||||
|
|
||||||
|
```go
|
||||||
|
if err := validateWhereClauseSecurity(where); err != nil {
|
||||||
|
logger.Debug("Security validation failed for WHERE clause: %v", err)
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
Every caller treats `""` as "no filter" and skips `query.Where`. So a clause
|
||||||
|
the sanitizer rejects is **removed**, and the request runs unfiltered instead
|
||||||
|
of failing. The denylist is a substring match on the whole clause, string
|
||||||
|
literals included, so ordinary filters trip it. The probe showed
|
||||||
|
`status = 'awaiting update approval'` and `last_update > '2020-01-01'` both
|
||||||
|
returning `""`, which gives an unfiltered list. For a preload `where` or a
|
||||||
|
cursor filter, that means returning rows the client asked to exclude, or
|
||||||
|
breaking pagination.
|
||||||
|
|
||||||
|
**Recommendation.** Return an error and make the handler respond `400`. Never
|
||||||
|
turn a rejected filter into "no filter".
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### 7. Medium — `cql*` columns bypass column validation
|
||||||
|
|
||||||
|
`validation.go:107-110` accepts any column that starts with `cql`
|
||||||
|
(case-insensitive), with no further checks. The probe showed
|
||||||
|
`IsValidColumn("cql1); drop")` returning `true`. The computed-column mechanism
|
||||||
|
only ever generates `cql1…cqlN` (`restheadspec/headers.go:871, 1380`). Whether a
|
||||||
|
client-supplied `cql…` string reaches SQL unquoted depends on the downstream
|
||||||
|
handler (`restheadspec/handler.go:490-509`, `cursor.go:189`). The validator
|
||||||
|
shouldn't be where that decision is made.
|
||||||
|
|
||||||
|
**Recommendation.** Accept only `^cql[0-9]+$`, and only when that computed
|
||||||
|
column was actually registered for the request.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### 8. Medium — Unbounded request body reads
|
||||||
|
|
||||||
|
`adapters/router/mux.go:101-115` uses `io.ReadAll(h.req.Body)` with no
|
||||||
|
`http.MaxBytesReader`, and `bunrouter.go:102-115` delegates to it. The
|
||||||
|
request-size middleware exists but isn't mounted (`_CROSS-CUTTING.audit.md`
|
||||||
|
X10), so a single request can make the process buffer gigabytes.
|
||||||
|
|
||||||
|
**Recommendation.** Wrap the body in `http.MaxBytesReader` inside the adapter,
|
||||||
|
with a configurable limit (for example 10 MB) and a sensible default.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### 9. Medium — Sensitive data in error logs
|
||||||
|
|
||||||
|
- `bun.go:1311-1315` (and the equivalent in `ScanModel`/`Count`, and in
|
||||||
|
`pgsql.go` / `gorm.go`) logs `b.query.String()`, the SQL with **all argument
|
||||||
|
values interpolated**, at `Error` on every failed query. That includes
|
||||||
|
filter values, emails, and tokens used as lookup keys.
|
||||||
|
- `recursive_crud.go` logs `data=%+v` (whole rows, including password or secret
|
||||||
|
columns) at `Error` on every failed nested write (lines 121, 153, 312, 342,
|
||||||
|
352, 509, 528, 550).
|
||||||
|
- `logger.Error` is forwarded to Sentry unscrubbed (`_CROSS-CUTTING.audit.md`
|
||||||
|
X8), and a hostile client can trigger failing queries at will.
|
||||||
|
|
||||||
|
**Recommendation.** Log the query with placeholders, not interpolated. Log
|
||||||
|
column names, not values. Put full dumps behind a debug flag.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### 10. Low — Fragile SQL string rewriting
|
||||||
|
|
||||||
|
- `reEmptyCompMid` / `reEmptyCompEnd` (`sql_helpers.go:66-80`) run over the
|
||||||
|
whole SQL string, including string literals and subqueries, and silently
|
||||||
|
delete text that matches `col = and`. That can change a query's meaning.
|
||||||
|
- The quote tracking in `splitByAND` / `findOperatorOutsideParentheses` /
|
||||||
|
`stripWrappingParens` toggles on every `'`, so an escaped `''` inside a
|
||||||
|
literal flips the state.
|
||||||
|
- `qualifyColumnInCondition` (`sql_helpers.go:751-760`) compiles a regex on
|
||||||
|
every call in the per-request path.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### 11. Low — Unsynchronised adapter field reads
|
||||||
|
|
||||||
|
`BunAdapter` protects `db` with `dbMu` in `getDB`/`reconnectDB`. But
|
||||||
|
`NewSelect` stores `db: b.db` (`bun.go:170`, used for count queries), and
|
||||||
|
`DriverName` reads `b.db` (`bun.go:280`), both without the lock.
|
||||||
|
`PgSQLAdapter.GetUnderlyingDB` (`pgsql.go:220`) does the same. These are data
|
||||||
|
races with `reconnectDB`, and `-race` would flag them
|
||||||
|
(`_CROSS-CUTTING.audit.md` X1). In practice the count query can run against
|
||||||
|
the old, closed pool.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### 12. Info — JSON column parsing is sound
|
||||||
|
|
||||||
|
`json_column.go` / `json_condition.go` are a good model for how the rest of
|
||||||
|
this package should handle client input:
|
||||||
|
|
||||||
|
- The base column must match `^[A-Za-z_][A-Za-z0-9_]*$` and is quoted with
|
||||||
|
`QuoteIdent`.
|
||||||
|
- Casts go through an allowlist.
|
||||||
|
- The JSON path is always bound as one `?::text[]` parameter, with depth and
|
||||||
|
segment-size limits.
|
||||||
|
- The dotted shorthand counts as JSON only when reflection confirms that the
|
||||||
|
base is a JSON column.
|
||||||
|
- The alias is validated and quoted.
|
||||||
|
|
||||||
|
No findings.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Panic handling
|
||||||
|
|
||||||
|
The adapter methods (`Scan`, `ScanModel`, `Count`, `Exec`, `Query`,
|
||||||
|
`RunInTransaction`) recover and convert panics with `logger.HandlePanic`.
|
||||||
|
`PgSQLAdapter.RunInTransaction` rolls back on panic before re-raising or
|
||||||
|
converting it. `BunAdapter.RunInTransaction` relies on bun's `RunInTx`, which
|
||||||
|
also rolls back. No panic paths were found in `sql_helpers.go`,
|
||||||
|
`validation.go` or the JSON parser that are reachable from client input.
|
||||||
|
Slicing is length-guarded. `recursive_crud.go` recurses over the model's
|
||||||
|
relation graph. For self-referential models, depth is bounded only by the
|
||||||
|
JSON decoder's nesting limit, which makes it a slowness issue rather than a
|
||||||
|
crash.
|
||||||
|
|
||||||
|
## Test coverage
|
||||||
|
|
||||||
|
`sql_helpers_test.go` and `validation_test.go` test the *intended* behaviour
|
||||||
|
of the sanitizer and validator. None of them test hostile inputs. Each probe
|
||||||
|
case in findings 1, 3, 6 and 7 is a one-line table entry and should be added
|
||||||
|
as a regression test that asserts rejection. `cors.go` and `recursive_crud.go`
|
||||||
|
have no security-focused tests.
|
||||||
@@ -0,0 +1,419 @@
|
|||||||
|
# Audit — `pkg/config`
|
||||||
|
|
||||||
|
- **Date:** 2026-09-29
|
||||||
|
- **Scope:** `pkg/config/{config,dbmanager,manager,paths,server}.go` (1023 LOC source, 608 LOC tests)
|
||||||
|
- **Axes:** thread locking/waiting · slowness · security · panic handling & logging
|
||||||
|
- **Threat model:** hostile internet client. Config itself is operator-controlled, so the security
|
||||||
|
focus here is **insecure defaults that the internet-facing layers inherit**, plus secret handling.
|
||||||
|
|
||||||
|
## Summary
|
||||||
|
|
||||||
|
Viper-backed configuration with a singleton `Manager`, a large `setDefaults` table, and per-section
|
||||||
|
validators. Two serious issues:
|
||||||
|
|
||||||
|
1. **`Manager` is a data race by construction.** It wraps a `*viper.Viper`, which has **no internal
|
||||||
|
locking** (verified: no `sync.Mutex`/`RWMutex` anywhere in `viper@v1.21.0/viper.go`'s `Viper`
|
||||||
|
struct), and exposes `Get`/`Set` as concurrently-callable methods on an unsynchronised lazy
|
||||||
|
singleton. A concurrent `Set` + `Get` is a concurrent map write → **`fatal error`, not a
|
||||||
|
recoverable panic**.
|
||||||
|
2. **The default configuration is insecure on every axis that matters** — wildcard CORS,
|
||||||
|
`sslmode=disable`, `user: postgres` with a blank password — and `Load()` silently succeeds when
|
||||||
|
no config file is found, so a misdeployment lands on exactly those defaults with no warning.
|
||||||
|
|
||||||
|
| # | Severity | Axis | Finding |
|
||||||
|
|---|----------|------|---------|
|
||||||
|
| 1 | **Critical** | Locking | `Manager.Set`/`Get` over a lock-free `*viper.Viper` → concurrent map write → process-fatal |
|
||||||
|
| 2 | **High** | Locking | `GetConfigManager()` is an unsynchronised lazy singleton; `NewManager()` also clobbers the global as a side effect |
|
||||||
|
| 3 | **High** | Security | **OPEN (deferred)** Insecure defaults: `cors.allowed_origins: ["*"]`, `allowed_headers: ["*"]`, `sslmode: disable`, `user: postgres` + blank password |
|
||||||
|
| 4 | **High** | Security | `SaveConfig` writes all secrets in plaintext at mode `0644` (viper default, never overridden) |
|
||||||
|
| 5 | Medium | Security | `AddConfigPath(".")` is searched first — CWD config injection |
|
||||||
|
| 6 | Medium | Observability | `Load()` swallows `ConfigFileNotFoundError` with no log at all |
|
||||||
|
| 7 | Medium | Correctness | `PathsConfig.Set` on a nil map panics; every sibling method nil-guards |
|
||||||
|
| 8 | Medium | Locking | `PathsConfig` is a bare `map[string]string` with a mutating `Set` — concurrent access is process-fatal |
|
||||||
|
| 9 | Medium | Slowness | `GetIPs()` does an uncontexted `net.LookupIP` — blocks on the resolver timeout |
|
||||||
|
| 10 | Medium | Correctness | `SetConfig` does a pointless `Unmarshal` into a discarded map whose error fails the call |
|
||||||
|
| 11 | Low | Panic | `GetIPs()` recovers to `fmt.Println`, bypassing the logger, and returns zeroed named results |
|
||||||
|
| 12 | Low | Security | No validation of `middleware.*` / `event_broker.worker_count` — `0` workers is accepted |
|
||||||
|
| 13 | Low | Correctness | `ServersConfig.GetDefault()` returns a pointer to a copy of a map value |
|
||||||
|
| 14 | Low | Security | `PathsConfig.Join` does not confine the result to the base path |
|
||||||
|
|
||||||
|
## Resolution status (2026-09-30)
|
||||||
|
|
||||||
|
- **#1** — Fixed: `sync.RWMutex` guards every viper access, options included
|
||||||
|
- **#2** — Fixed: mutex-guarded singleton; `NewManager` no longer touches the global (new `SetConfigManager` publishes explicitly)
|
||||||
|
- **#4** — Fixed: `SetConfigPermissions(0o600)` plus `chmod 0600` after write (secrets are not stripped)
|
||||||
|
- **#5** — Fixed: search order is `/etc/resolvespec`, `$HOME/.resolvespec`, `./config`, `.` (CWD last, not dropped)
|
||||||
|
- **#6** — Partly fixed: `ConfigFileUsed()` added; no log line because `pkg/config` cannot import `logger` (import cycle)
|
||||||
|
- **#7** — Fixed: `Set` has a pointer receiver and allocates
|
||||||
|
- **#8** — Not fixed: still a bare map; `Set` documented as not concurrency-safe
|
||||||
|
- **#9** — Fixed: `LookupIPAddr` with a 2s timeout, fallback normalised to bare IPs and populates the slice
|
||||||
|
- **#10** — Fixed: dead `Unmarshal` removed, `SetConfig` is atomic
|
||||||
|
- **#11** — Fixed: recover removed (nothing in the function can panic)
|
||||||
|
- **#12** — Partly fixed: `Config.Validate()` added, but it is not called from `GetConfig()`. The `*` CORS+credentials check is not implemented
|
||||||
|
- **#13** — Documented only: `GetDefault` returns a pointer to a copy
|
||||||
|
- **#14** — Fixed: `Join` errors if the result escapes the base
|
||||||
|
- **#3** — Open: default flips deferred by decision (breaking change).
|
||||||
|
- Tests: `pkg/config/hardening_test.go`.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Findings
|
||||||
|
|
||||||
|
### 1. `Manager` exposes a lock-free viper as a concurrent API (Critical, Locking)
|
||||||
|
|
||||||
|
`manager.go:10-13`, `manager.go:133-158`
|
||||||
|
|
||||||
|
```go
|
||||||
|
type Manager struct {
|
||||||
|
v *viper.Viper
|
||||||
|
}
|
||||||
|
...
|
||||||
|
func (m *Manager) Get(key string) interface{} { return m.v.Get(key) }
|
||||||
|
func (m *Manager) GetString(key string) string { return m.v.GetString(key) }
|
||||||
|
func (m *Manager) Set(key string, value interface{}) { m.v.Set(key, value) }
|
||||||
|
```
|
||||||
|
|
||||||
|
`viper.Viper` carries its configuration in plain maps (`override`, `config`, `defaults`, `aliases`,
|
||||||
|
…) and has **no mutex**. Verified against the module in use:
|
||||||
|
|
||||||
|
```
|
||||||
|
$ grep -n 'sync\.\|Lock()' $(go env GOMODCACHE)/github.com/spf13/viper@v1.21.0/viper.go
|
||||||
|
319: initWG := sync.WaitGroup{} # inside WatchConfig only
|
||||||
|
340: eventsWG := sync.WaitGroup{} # inside WatchConfig only
|
||||||
|
```
|
||||||
|
|
||||||
|
`Set` writes to `v.override`; `Get` reads across those maps. Because `GetConfigManager()` hands the
|
||||||
|
*same* `*Manager` to every caller, any code path that calls `Manager.Set` at runtime while another
|
||||||
|
goroutine reads config is a concurrent map read/write. Go's runtime detects this and issues
|
||||||
|
`fatal error: concurrent map read and map write` — which **`recover()` cannot catch**, so none of
|
||||||
|
the panic handlers elsewhere in the codebase will save the process.
|
||||||
|
|
||||||
|
This is latent-but-loaded: it needs one runtime `Set` to become a crash. `SetConfig`
|
||||||
|
(`manager.go:107-131`) performs eleven `m.v.Set` calls, so any dynamic reconfiguration triggers it.
|
||||||
|
|
||||||
|
**Recommendation:** add a `sync.RWMutex` to `Manager` and take it in every method that touches
|
||||||
|
`m.v` (including the `Option` functions at `manager.go:60-85`, which also mutate viper). Better:
|
||||||
|
load once into an immutable `*Config` at startup and pass that value around, keeping `Manager`
|
||||||
|
confined to startup.
|
||||||
|
|
||||||
|
### 2. Unsynchronised lazy singleton (High, Locking)
|
||||||
|
|
||||||
|
`manager.go:15-45`
|
||||||
|
|
||||||
|
```go
|
||||||
|
var configInstance *Manager
|
||||||
|
|
||||||
|
func GetConfigManager() *Manager {
|
||||||
|
if configInstance == nil {
|
||||||
|
configInstance = NewManager()
|
||||||
|
}
|
||||||
|
return configInstance
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
Classic check-then-act race: two concurrent first calls both see `nil`, both build a `Manager`,
|
||||||
|
and the two callers get *different* instances — so a `Set` through one is invisible through the
|
||||||
|
other. The unsynchronised pointer write races with the read.
|
||||||
|
|
||||||
|
Worse, `NewManager()` (`manager.go:27-45`) assigns `configInstance = &Manager{v: v}` at line 43 as
|
||||||
|
a **side effect**. So a caller who deliberately builds an isolated manager silently replaces the
|
||||||
|
global one, and `NewManagerWithOptions` (`manager.go:48-54`) publishes a half-configured manager to
|
||||||
|
the global *before* applying its options — another goroutine can observe the instance mid-mutation.
|
||||||
|
|
||||||
|
**Recommendation:** `sync.Once` for the singleton; remove the global assignment from `NewManager`.
|
||||||
|
|
||||||
|
### 3. Insecure-by-default configuration (High, Security)
|
||||||
|
|
||||||
|
`manager.go:203-247`:
|
||||||
|
|
||||||
|
```go
|
||||||
|
v.SetDefault("cors.allowed_origins", []string{"*"})
|
||||||
|
v.SetDefault("cors.allowed_headers", []string{"*"})
|
||||||
|
...
|
||||||
|
v.SetDefault("dbmanager.connections.default.user", "postgres")
|
||||||
|
v.SetDefault("dbmanager.connections.default.password", "")
|
||||||
|
v.SetDefault("dbmanager.connections.default.sslmode", "disable")
|
||||||
|
```
|
||||||
|
|
||||||
|
Each of these is inherited by an internet-facing layer:
|
||||||
|
|
||||||
|
- **`allowed_origins: ["*"]` + `allowed_headers: ["*"]`** — any origin may make cross-origin calls
|
||||||
|
with arbitrary headers. Whether this is exploitable depends on whether the CORS middleware also
|
||||||
|
sets `Access-Control-Allow-Credentials`; see `audit/pkg/middleware.audit.md` for that
|
||||||
|
determination. Even without credentials, wildcard origin plus wildcard headers defeats any
|
||||||
|
header-based CSRF defence and lets a malicious page read responses from a
|
||||||
|
network-position-authenticated deployment (IP allowlisted, mTLS-terminated, VPN).
|
||||||
|
- **`sslmode: disable`** — DB traffic unencrypted by default. Every row that crosses the wire,
|
||||||
|
including whatever the internet-facing handlers select, is plaintext on the network.
|
||||||
|
- **`user: postgres` with an empty password** — the default connection targets the PostgreSQL
|
||||||
|
superuser. Combined with the identifier-handling concerns in
|
||||||
|
`audit/pkg/common.audit.md` / `audit/pkg/restheadspec.audit.md`, running as superuser removes the
|
||||||
|
last line of defence (least-privilege) against a query-construction bug.
|
||||||
|
|
||||||
|
Because of finding 6, a deployment with a missing or misnamed config file runs on **all** of these
|
||||||
|
simultaneously and reports success.
|
||||||
|
|
||||||
|
**Recommendation:** default to `sslmode: require`, no default DB user/password (fail loudly if
|
||||||
|
unset), and `cors.allowed_origins: []` with wildcard requiring an explicit opt-in. Add a
|
||||||
|
`Config.Validate()` that refuses `allowed_origins: ["*"]` together with credentials.
|
||||||
|
|
||||||
|
### 4. `SaveConfig` writes secrets in plaintext at 0644 (High, Security)
|
||||||
|
|
||||||
|
`manager.go:160-166`
|
||||||
|
|
||||||
|
```go
|
||||||
|
func (m *Manager) SaveConfig(path string) error {
|
||||||
|
if err := m.v.WriteConfigAs(path); err != nil { ... }
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
`WriteConfigAs` serialises the **entire** merged configuration. That includes
|
||||||
|
`dbmanager.connections.*.password`, `cache.redis.password`, `event_broker.redis.password` and
|
||||||
|
`error_tracking.dsn` (a Sentry DSN is a credential).
|
||||||
|
|
||||||
|
Viper writes with `v.configPermissions`, which defaults to `0o644`
|
||||||
|
(`viper@v1.21.0/viper.go:198`). `SetConfigPermissions` is **never called anywhere in this repo**
|
||||||
|
(verified by grep), so the file is world-readable. Any local user or any other container sharing
|
||||||
|
the mount can read the DB superuser password.
|
||||||
|
|
||||||
|
**Recommendation:** call `v.SetConfigPermissions(0o600)` in `NewManager`; better, strip secret keys
|
||||||
|
before writing and document that secrets come from env/secret-manager only.
|
||||||
|
|
||||||
|
### 5. Current-working-directory config injection (Medium, Security)
|
||||||
|
|
||||||
|
`manager.go:32-36`
|
||||||
|
|
||||||
|
```go
|
||||||
|
v.AddConfigPath(".")
|
||||||
|
v.AddConfigPath("./config")
|
||||||
|
v.AddConfigPath("/etc/resolvespec")
|
||||||
|
v.AddConfigPath("$HOME/.resolvespec")
|
||||||
|
```
|
||||||
|
|
||||||
|
Viper searches these **in order** and takes the first hit, so `./config.yaml` wins over
|
||||||
|
`/etc/resolvespec/config.yaml`. For a daemon this is backwards: the CWD is the least trustworthy of
|
||||||
|
the four. If the process is ever started with its CWD in a shared or user-writable directory (a
|
||||||
|
tmp dir, a bind-mounted volume, `/` in some container setups), an attacker with local write
|
||||||
|
capability redirects the DB connection, disables TLS, or points `error_tracking.dsn` at their own
|
||||||
|
collector — turning finding 1 of `audit/pkg/errortracking.audit.md` into a full exfiltration path.
|
||||||
|
|
||||||
|
**Recommendation:** search `/etc/resolvespec` first, drop `"."` from the default list (keep it
|
||||||
|
available via `WithConfigPath`), and log the resolved path at startup (`v.ConfigFileUsed()`).
|
||||||
|
|
||||||
|
### 6. `Load()` is silent about a missing config file (Medium, Observability)
|
||||||
|
|
||||||
|
`manager.go:87-97`
|
||||||
|
|
||||||
|
```go
|
||||||
|
if err := m.v.ReadInConfig(); err != nil {
|
||||||
|
if _, ok := err.(viper.ConfigFileNotFoundError); !ok {
|
||||||
|
return fmt.Errorf("error reading config file: %w", err)
|
||||||
|
}
|
||||||
|
// Config file not found; will rely on defaults and env vars
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
```
|
||||||
|
|
||||||
|
The comment is the only trace. No log line, no returned indicator, no `ConfigFileUsed()` report.
|
||||||
|
A typo in the filename, a wrong working directory, or a container that forgot to mount the
|
||||||
|
ConfigMap is indistinguishable from a deliberate defaults-only run — and the defaults are the ones
|
||||||
|
in finding 3.
|
||||||
|
|
||||||
|
**Recommendation:** log at info level whether a file was used and which one; expose
|
||||||
|
`ConfigFileUsed()` on `Manager` so startup can print it.
|
||||||
|
|
||||||
|
### 7. `PathsConfig.Set` panics on a nil map (Medium, Panic handling)
|
||||||
|
|
||||||
|
`paths.go:38-40`
|
||||||
|
|
||||||
|
```go
|
||||||
|
func (pc PathsConfig) Set(name, path string) {
|
||||||
|
pc[name] = path
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
`PathsConfig` is `map[string]string` (`config.go:200`). `Get`, `GetOrDefault`, `Has` and `List` all
|
||||||
|
begin with `if pc == nil`. `Set` does not — and assignment to a nil map is
|
||||||
|
`panic: assignment to entry in nil map`.
|
||||||
|
|
||||||
|
`Config.Paths` is populated by `mapstructure`, which leaves the map nil when the `paths` key is
|
||||||
|
absent from the file. `setDefaults` does register `paths.data_dir` etc. (`manager.go:249-253`), so
|
||||||
|
the map is non-nil on the normal `GetConfig()` path — but a `Config` built in code
|
||||||
|
(`config.Config{}`) or produced by a partial unmarshal has a nil `Paths`, and `Set` on it panics.
|
||||||
|
Nothing in `pkg/` currently calls `Set` (verified by grep), so this is a latent API defect.
|
||||||
|
|
||||||
|
**Recommendation:** nil-guard consistently, or change the receiver to `*PathsConfig` so `Set` can
|
||||||
|
allocate.
|
||||||
|
|
||||||
|
### 8. `PathsConfig` has no synchronisation (Medium, Locking)
|
||||||
|
|
||||||
|
Same type: a bare map with a mutating `Set` and reading `Get`/`Has`/`List`/`EnsureDir`/`AbsPath`/
|
||||||
|
`Join`. If any consumer calls `Set` at runtime while request handlers resolve paths, that is a
|
||||||
|
concurrent map write — again the **unrecoverable** `fatal error` class, not a panic.
|
||||||
|
|
||||||
|
Currently unused outside the package, so severity is capped at Medium. If the intent is a runtime
|
||||||
|
path registry, it needs a mutex and an unexported map.
|
||||||
|
|
||||||
|
### 9. `GetIPs()` blocks on an uncontexted DNS lookup (Medium, Slowness)
|
||||||
|
|
||||||
|
`server.go:113-149`
|
||||||
|
|
||||||
|
```go
|
||||||
|
hostname, _ = os.Hostname()
|
||||||
|
...
|
||||||
|
addrs, err := net.LookupIP(hostname)
|
||||||
|
```
|
||||||
|
|
||||||
|
`net.LookupIP` has no context and no timeout override — it blocks for the resolver's own timeout,
|
||||||
|
which on a misconfigured or slow-resolver host is 5 s per attempt and up to ~15–20 s with retries
|
||||||
|
across `/etc/resolv.conf` entries. In a container whose hostname is not in DNS (the normal case)
|
||||||
|
this fails, but only *after* the resolver gives up.
|
||||||
|
|
||||||
|
There is no caller in `pkg/` today, so it is not on the request path yet. It is exported and
|
||||||
|
named like a utility, so the risk is that it lands on one.
|
||||||
|
|
||||||
|
Secondary correctness problem in the same function: the fallback branch (`server.go:139-147`)
|
||||||
|
appends `a.String()` for a `net.Addr` from `net.InterfaceAddrs()`, which renders as CIDR
|
||||||
|
(`192.168.1.5/24`), into the same comma-joined string that the primary branch fills with bare IPs.
|
||||||
|
Consumers get two formats from one field. That branch also never appends to `ipaddrlist`, so the
|
||||||
|
third return value is empty whenever the fallback is taken.
|
||||||
|
|
||||||
|
**Recommendation:** `net.DefaultResolver.LookupIPAddr(ctx, host)` with a short deadline; cache the
|
||||||
|
result; normalise the fallback to bare IPs via `net.Addr.(*net.IPNet).IP`.
|
||||||
|
|
||||||
|
### 10. `SetConfig` does dead work that can fail the call (Medium, Correctness)
|
||||||
|
|
||||||
|
`manager.go:107-131`
|
||||||
|
|
||||||
|
```go
|
||||||
|
configMap := make(map[string]interface{})
|
||||||
|
if err := m.v.Unmarshal(&configMap); err != nil {
|
||||||
|
return fmt.Errorf("failed to prepare config map: %w", err)
|
||||||
|
}
|
||||||
|
// configMap is never read again
|
||||||
|
m.v.Set("servers", cfg.Servers)
|
||||||
|
...
|
||||||
|
```
|
||||||
|
|
||||||
|
`configMap` is written and then never used. The comment says "Marshal the config to a map structure
|
||||||
|
that viper can use", but it unmarshals *viper's current state* into a throwaway map — it has
|
||||||
|
nothing to do with `cfg`. The only effect is that a decode error in the **existing** config makes
|
||||||
|
`SetConfig` fail for no reason. It also does a full reflective decode of the whole config tree on
|
||||||
|
every call.
|
||||||
|
|
||||||
|
Note also that `SetConfig` stores Go structs into viper via `Set`, and the eleven `Set` calls are
|
||||||
|
not atomic — a concurrent `GetConfig()` observes a torn config (new `servers`, old `cors`), on top
|
||||||
|
of finding 1's race.
|
||||||
|
|
||||||
|
**Recommendation:** delete the `configMap` block.
|
||||||
|
|
||||||
|
### 11. `GetIPs()` panic handling bypasses the logger (Low, Panic handling)
|
||||||
|
|
||||||
|
`server.go:114-118`
|
||||||
|
|
||||||
|
```go
|
||||||
|
defer func() {
|
||||||
|
if err := recover(); err != nil {
|
||||||
|
fmt.Println("Recovered in GetIPs", err)
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
```
|
||||||
|
|
||||||
|
- Writes to stdout with `fmt.Println` rather than `logger.Error`/`logger.HandlePanic`, so the event
|
||||||
|
never reaches the error tracker and is invisible to structured log collection.
|
||||||
|
- No stack trace captured.
|
||||||
|
- The function's results are named (`hostname, ipList string, ipNetList []net.IP`) but the body
|
||||||
|
builds `iplist`/`ipaddrlist` **locals** and only assigns via the `return` statements. On a panic,
|
||||||
|
the deferred recover swallows it and the function returns the *zero* named values — `ipNetList`
|
||||||
|
is nil rather than the empty slice callers might expect. Silent empty success.
|
||||||
|
|
||||||
|
`pkg/config` is otherwise the only package outside `pkg/logger` that hand-rolls a recover instead
|
||||||
|
of using the shared helpers.
|
||||||
|
|
||||||
|
**Recommendation:** use `defer logger.CatchPanic("GetIPs")()`, or drop the recover — there is no
|
||||||
|
panicking operation in this function for it to catch.
|
||||||
|
|
||||||
|
### 12. No validation of numeric/limit settings (Low, Security)
|
||||||
|
|
||||||
|
`ServerInstanceConfig.Validate` (`server.go:37-68`) and `ServersConfig.Validate`
|
||||||
|
(`server.go:71-95`) are good — port range, mutually-exclusive TLS modes, cert/key pairing,
|
||||||
|
AutoTLS domains. But nothing validates:
|
||||||
|
|
||||||
|
- `middleware.rate_limit_rps` / `rate_limit_burst` — `0` disables rate limiting silently.
|
||||||
|
- `middleware.max_request_size` — `0` may mean unlimited depending on the middleware; see
|
||||||
|
`audit/pkg/middleware.audit.md`.
|
||||||
|
- `event_broker.worker_count` (default 10) — `0` means no consumers; see
|
||||||
|
`audit/pkg/eventbroker.audit.md` for whether that deadlocks publishers or drops events.
|
||||||
|
- `dbmanager.max_open_conns`, retry counts/delays — negative or zero values.
|
||||||
|
- `cors.allowed_origins: ["*"]` in combination with credentials.
|
||||||
|
|
||||||
|
There is also no top-level `Config.Validate()` that calls the section validators, so nothing
|
||||||
|
guarantees `ServersConfig.Validate` ever runs.
|
||||||
|
|
||||||
|
**Recommendation:** add `func (c *Config) Validate() error` that fans out to every section, and
|
||||||
|
call it from `GetConfig()`.
|
||||||
|
|
||||||
|
### 13. `GetDefault()` returns a pointer to a copy (Low, Correctness)
|
||||||
|
|
||||||
|
`server.go:98-110`
|
||||||
|
|
||||||
|
```go
|
||||||
|
instance, ok := sc.Instances[sc.DefaultServer]
|
||||||
|
...
|
||||||
|
return &instance, nil
|
||||||
|
```
|
||||||
|
|
||||||
|
`instance` is a copy of the map value. A caller that mutates through the returned pointer — which
|
||||||
|
the `*ServerInstanceConfig` receiver on `ApplyGlobalDefaults` (`server.go:12`) invites — changes
|
||||||
|
only the copy, and `sc.Instances` is unaffected. This is exactly the shape of bug where timeouts
|
||||||
|
appear to be applied but aren't.
|
||||||
|
|
||||||
|
**Recommendation:** make `Instances` a `map[string]*ServerInstanceConfig`, or return by value.
|
||||||
|
|
||||||
|
### 14. `PathsConfig.Join` does not confine to the base (Low, Security)
|
||||||
|
|
||||||
|
`paths.go:96-104`
|
||||||
|
|
||||||
|
```go
|
||||||
|
parts := append([]string{base}, elem...)
|
||||||
|
return filepath.Join(parts...), nil
|
||||||
|
```
|
||||||
|
|
||||||
|
`filepath.Join` calls `Clean`, which *resolves* `..` rather than rejecting it: `Join("data",
|
||||||
|
"../../etc/passwd")` returns `../etc/passwd`. Any consumer that passes a request-derived segment
|
||||||
|
gets directory traversal out of the configured base. No consumer does today, hence Low, but the
|
||||||
|
method's name promises confinement it does not provide.
|
||||||
|
|
||||||
|
**Recommendation:** after joining, verify `strings.HasPrefix(filepath.Clean(result), filepath.Clean(base)+string(os.PathSeparator))`, or use `os.Root`/`filepath.Localize` on the elements.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## What looks right
|
||||||
|
|
||||||
|
- `ServerInstanceConfig.Validate` / `ServersConfig.Validate` (`server.go:37-95`) are thorough:
|
||||||
|
port bounds, mutual exclusion of the three TLS modes, cert/key co-presence, AutoTLS domain
|
||||||
|
requirement, and a key-vs-`Name` consistency check on the instances map. This is the strongest
|
||||||
|
code in the package.
|
||||||
|
- `ApplyGlobalDefaults` (`server.go:12-32`) uses `*time.Duration` fields so "unset" is
|
||||||
|
distinguishable from "zero" — the right modelling choice, and it copies into a fresh local
|
||||||
|
before taking its address rather than aliasing the loop/parameter variable.
|
||||||
|
- `Load()` correctly distinguishes `ConfigFileNotFoundError` from real read errors instead of
|
||||||
|
treating every failure as fatal (the *silence* is the problem, not the branch).
|
||||||
|
- `SetEnvPrefix("RESOLVESPEC")` + `SetEnvKeyReplacer(".", "_")` + `AutomaticEnv`
|
||||||
|
(`manager.go:38-41`) is the correct trio for env overrides, and because every key has a
|
||||||
|
registered default, `AutomaticEnv` actually resolves nested keys — so secrets *can* be supplied
|
||||||
|
via env instead of the file. That's the mitigation for finding 4, and it should be documented as
|
||||||
|
the only supported way to pass secrets.
|
||||||
|
- The defaults table is comprehensive and one place — easy to review, which is how findings 3 and
|
||||||
|
12 were found.
|
||||||
|
- Test coverage is reasonable for a config package (608 LOC of tests against 1023 of source),
|
||||||
|
though it does not cover concurrency, `SaveConfig` permissions, or `PathsConfig.Set`.
|
||||||
|
|
||||||
|
## Suggested follow-up
|
||||||
|
|
||||||
|
1. Lock `Manager` or make config immutable after load (findings 1, 2). Until then, treat
|
||||||
|
`Manager.Set` as unsafe to call after startup and consider removing it from the public API.
|
||||||
|
2. Flip the insecure defaults and add `Config.Validate()` (findings 3, 12).
|
||||||
|
3. `SetConfigPermissions(0o600)` and secret-stripping in `SaveConfig` (finding 4).
|
||||||
|
4. Reorder the config search path and log the resolved file (findings 5, 6).
|
||||||
|
5. Delete the dead `Unmarshal` in `SetConfig` (finding 10).
|
||||||
@@ -0,0 +1,630 @@
|
|||||||
|
# Audit: `pkg/dbmanager`
|
||||||
|
|
||||||
|
| | |
|
||||||
|
|---|---|
|
||||||
|
| **Package** | `github.com/bitechdev/ResolveSpec/pkg/dbmanager` (+ `providers/`) |
|
||||||
|
| **Files** | `config.go` (489), `connection.go` (722), `manager.go` (401), `metrics.go` (136), `errors.go` (82), `factory.go` (67), `providers/postgres.go` (231), `providers/postgres_listener.go` (401), `providers/sqlite.go` (216), `providers/mongodb.go` (214), `providers/mssql.go` (184), `providers/existing_db.go` (111), `providers/provider.go` (89); tests `factory_test.go` (369), `manager_test.go` (290), `providers/existing_db_test.go` (194), `providers/postgres_listener_example_test.go` (229) |
|
||||||
|
| **Audit date** | 2026-09-30 |
|
||||||
|
| **Axes** | thread locking/waiting, slowness, security, panic handling & logging |
|
||||||
|
| **Threat model** | hostile internet client; request bodies, headers, query params, schema/table/column names all attacker-controlled |
|
||||||
|
| **Depth** | deep (hot package; every request's DB handle comes from here). Several findings were checked with a throw-away probe test against SQLite, and the probe was deleted afterwards |
|
||||||
|
|
||||||
|
## Summary
|
||||||
|
|
||||||
|
`pkg/dbmanager` owns every database pool in the process. It wraps a
|
||||||
|
`*sql.DB` (or a `mongo.Client`) in a `sqlConnection` and hands out lazily-built
|
||||||
|
`*bun.DB`, `*gorm.DB`, raw `*sql.DB` and `common.Database` adapters over it. A
|
||||||
|
background health checker pings each connection every 15 s, and it can
|
||||||
|
**reconnect**, which closes the pool and opens a new one.
|
||||||
|
|
||||||
|
This audit was started to answer one question: **"why does a database
|
||||||
|
connection that has been idle for a while become unusable?"** Several defects
|
||||||
|
in this package combine to give exactly that symptom. They are findings 1–5,
|
||||||
|
and the [Idle-connection failure chain](#idle-connection-failure-chain) section
|
||||||
|
below puts them together.
|
||||||
|
|
||||||
|
The root design problem is that **`Reconnect` destroys the shared `*sql.DB`**.
|
||||||
|
`*sql.DB` is already a self-healing pool: it throws away bad connections and
|
||||||
|
dials new ones. So "reconnecting" a pool is almost never needed, and here it
|
||||||
|
has a large blast radius. Every `*bun.DB`, `*gorm.DB` and `*sql.DB` handed out
|
||||||
|
before the reconnect now points at a closed pool, and it stays closed. Only the
|
||||||
|
`common.Database` adapters carry a factory that can re-fetch a handle, and even
|
||||||
|
they only use it on a subset of code paths (see `common.audit.md` finding 5).
|
||||||
|
Those adapter factories also *trigger* `Reconnect` themselves, so one stale
|
||||||
|
handle closes the pool for everyone else. `Reconnect` isn't atomic, so
|
||||||
|
concurrent callers turn this into a storm.
|
||||||
|
|
||||||
|
The other major theme is **missing client-side deadlines**. `QueryTimeout` is
|
||||||
|
only ever sent to the server as `statement_timeout`, which does nothing when
|
||||||
|
the TCP peer has vanished. No `context.WithTimeout` is applied to request
|
||||||
|
queries, and pgx's dialer sets no `TCP_USER_TIMEOUT`. So the first query on a
|
||||||
|
pooled connection whose peer silently disappeared (NAT/firewall idle drop,
|
||||||
|
failover, a pgbouncer restart) can block for minutes. One Close path does this
|
||||||
|
while holding the connection's write lock, which stalls every request.
|
||||||
|
|
||||||
|
## Findings
|
||||||
|
|
||||||
|
| # | Severity | Axis | Finding | Status |
|
||||||
|
|---|---|---|---|---|
|
||||||
|
| 1 | **Critical** | locking / availability | `Reconnect` closes the shared `*sql.DB`, so every `*bun.DB` / `*gorm.DB` / `*sql.DB` handed out earlier is permanently dead ("sql: database is closed") | Fixed |
|
||||||
|
| 2 | **High** | locking | Adapter reconnect factories call `Reconnect` on the *shared* connection, and `Reconnect` is not atomic, so one stale handle starts a reconnect storm that repeatedly closes the pool under in-flight requests | Fixed |
|
||||||
|
| 3 | **High** | slowness / locking | `sqlConnection.HealthCheck` holds the write lock across a network ping for up to 5 s; every `Bun()`/`GORM()`/`Native()`/`Database()`/`Stats()` call blocks for that time | Fixed |
|
||||||
|
| 4 | **High** | slowness | No client-side query deadline and no `TCP_USER_TIMEOUT`: a query on a silently-dead idle socket blocks for minutes (up to about 15 min); `QueryTimeout` is server-side only, and is forced to at least 2 min | Fixed |
|
||||||
|
| 5 | **High** | locking / slowness | `PostgresListener.Close` runs `UNLISTEN` with `context.Background()` while `sqlConnection.mu` (write), `PostgresProvider.mu` and `listener.mu` are all held; on a dead socket this freezes every request for minutes | Fixed |
|
||||||
|
| 6 | **High** | locking / leak | `PostgresListener.Connect` starts a new goroutine pair on every (re)connect; the old pair keeps running, so two loops call `WaitForNotification` on one `pgx.Conn` concurrently, which triggers more reconnects | Fixed |
|
||||||
|
| 7 | **High** | panic handling | `Connect → Close → Connect → Close` panics with "close of closed channel"; after the first cycle the health checker also exits immediately and silently | Fixed |
|
||||||
|
| 8 | **Medium** | availability | SQLite: `:memory:` with a 25-connection pool gives every connection its own empty database, and `ConnMaxIdleTime` then silently discards data; `busy_timeout` / WAL pragmas are applied to only one pooled connection | Fixed |
|
||||||
|
| 9 | **Medium** | availability | Partial failure in `sqlConnection.Close` leaves `connected=true` over a closed pool; partial failure in `Manager.Connect` leaks the connections already opened | Fixed |
|
||||||
|
| 10 | **Medium** | security | DSN builders concatenate unescaped credentials (postgres key=value, mssql/mongo URLs); `sslmode` defaults to `disable` | Fixed |
|
||||||
|
| 11 | **Medium** | config | Several config knobs are ignored or impossible to turn off: `EnableAutoReconnect`, `HealthCheckInterval`, `RetryAttempts`/`RetryDelay`/`RetryMaxDelay`, SQLite `_timeout`, and `statement_timeout` when a DSN is given | Fixed |
|
||||||
|
| 12 | **Medium** | locking | `Manager.Connect` holds `m.mu` across every network dial (up to 3 retries × `ConnectTimeout` per connection) | Fixed |
|
||||||
|
| 13 | **Low** | observability | `PublishMetrics` / `RecordReconnectAttempt` are never called, so all dbmanager metrics are permanently zero; `*_total` metrics are gauges | Fixed |
|
||||||
|
| 14 | **Low** | correctness | `Bun()`/`GORM()` do not check `connected`; `getNativeAdapter` uses `PgSQLAdapter` for SQLite and MSSQL; `ExistingDBProvider` applies no pool settings and closes the caller's DB | Fixed (partly, see notes) |
|
||||||
|
| 15 | **Low** | logging | `Close` / `performHealthCheck` pass key-value pairs to the printf-style logger, which produces `%!(EXTRA ...)` output; `ResetInstance` discards the close error | Fixed |
|
||||||
|
|
||||||
|
## Remediation status
|
||||||
|
|
||||||
|
Implemented 2026-09-30. `go build ./...` and `go test -race ./pkg/dbmanager/...`
|
||||||
|
pass. The Postgres behaviour was also verified against a live server (tests are
|
||||||
|
skipped unless `PG_LIVE=1` / `PG_RESTART_DIR` is set).
|
||||||
|
|
||||||
|
**Design decisions taken**
|
||||||
|
- No automatic reconnect. Adapter factories and the health checker never close
|
||||||
|
the pool; they only re-fetch the current handle. `*sql.DB` replaces bad
|
||||||
|
connections itself. `EnableAutoReconnect` is deprecated and ignored.
|
||||||
|
- `Reconnect` is atomic (one critical section) and operator-only. On PostgreSQL
|
||||||
|
it goes through a custom `driver.Connector` (`providers/pgconnector.go`): it
|
||||||
|
bumps a generation, stale pooled connections are discarded, and the `*sql.DB`
|
||||||
|
is never closed, so held Bun/GORM/`*sql.DB` handles keep working. Other
|
||||||
|
providers still close and reopen.
|
||||||
|
- Client-side deadlines are applied at the driver level rather than in the
|
||||||
|
adapters (a `context.WithTimeout` around a query is cancelled before the
|
||||||
|
caller has read the rows).
|
||||||
|
|
||||||
|
**Per finding**
|
||||||
|
1. Fixed. Postgres refresh keeps the pool; explicit `Reconnect` on other
|
||||||
|
providers still invalidates handles (documented in the README).
|
||||||
|
2. Fixed. Adapter factories no longer call `Reconnect`; `Reconnect` is a single
|
||||||
|
critical section under `lifecycleMu` + `mu`.
|
||||||
|
3. Fixed. The ping runs without `mu`; `lifecycleMu` (read) only keeps
|
||||||
|
`Close`/`Reconnect` from tearing the provider down mid-ping. Same for Mongo.
|
||||||
|
4. Fixed. TCP keepalive and `TCP_USER_TIMEOUT` (30 s, Linux) via `DialFunc`;
|
||||||
|
the reuse-time liveness ping is capped at 5 s; `statement_timeout` is set as
|
||||||
|
a runtime parameter so it also applies to a supplied DSN; the 2-minute floor
|
||||||
|
on `QueryTimeout` is removed. `SetConnMaxIdleTime` tuning remains a
|
||||||
|
configuration matter (documented in the README).
|
||||||
|
5. Fixed. Listener `Close` sends no `UNLISTEN`, closes with a 2 s bound, and
|
||||||
|
holds no lock across network I/O.
|
||||||
|
6. Fixed. Background goroutines start once (`sync.Once`); reconnect dials a
|
||||||
|
replacement, re-`LISTEN`s, then swaps it in; sleeps honour `ctx.Done()`.
|
||||||
|
Additionally, all use of the single `pgx.Conn` is serialised (`connMu`, 500 ms
|
||||||
|
notification poll), fixing "conn busy" from `Listen`/`Unlisten`/`Notify`, and
|
||||||
|
old connections are closed under `connMu` (a race found by the live test).
|
||||||
|
7. Fixed. Stop channel is created per start, guarded by `healthMu`; `Close` is
|
||||||
|
idempotent; `Connect` is idempotent.
|
||||||
|
8. Fixed. `:memory:` is pinned to one connection with no idle/lifetime limits;
|
||||||
|
`busy_timeout`/WAL are `_pragma` DSN parameters; `_timeout` and the dead
|
||||||
|
reconnect code are removed.
|
||||||
|
9. Fixed. `Close` always marks disconnected and returns joined errors;
|
||||||
|
`PostgresProvider.Close` closes the pool even if the listener fails;
|
||||||
|
`Manager.Connect` closes connections it opened when a later one fails.
|
||||||
|
10. Fixed. Postgres, MSSQL and Mongo DSNs are built as escaped URLs; default
|
||||||
|
`sslmode` is now `prefer` (was `disable`).
|
||||||
|
11. Fixed. Retry settings reach every provider; a negative
|
||||||
|
`HealthCheckInterval` disables the health checker; `EnableAutoReconnect`
|
||||||
|
deprecated; `statement_timeout` applies with a supplied DSN.
|
||||||
|
12. Fixed. `Manager.Connect` dials outside `m.mu` and publishes results under it.
|
||||||
|
13. Fixed. `PublishMetrics` runs on each health-check tick, `Reconnect` records
|
||||||
|
`RecordReconnectAttempt`, and the wait/closed metrics are true counters
|
||||||
|
(delta-tracked).
|
||||||
|
14. Partly fixed. `Bun()`/`GORM()` check `connected`; Mongo no longer maps
|
||||||
|
`MaxIdleConns` to `MinPoolSize`. `ExistingDBProvider`: `Close` is now a no-op
|
||||||
|
that logs a warning (the caller owns the `*sql.DB`; the connection's `Close`
|
||||||
|
also skips `bun.DB.Close`), and `Reconnect` only pings. Pool settings are
|
||||||
|
still not applied to a caller-owned pool. The `getNativeAdapter` claim was
|
||||||
|
stale: the adapter already receives the driver name; the three duplicate
|
||||||
|
cases were merged. Mongo `Stats()` is still empty.
|
||||||
|
15. Fixed. Printf-style logger calls corrected; `ResetInstance` logs the close
|
||||||
|
error. Unscrubbed driver errors in Sentry (X8) are not addressed here.
|
||||||
|
|
||||||
|
**Behaviour changes**
|
||||||
|
- Removed tests that closed the pool from outside and expected an adapter to
|
||||||
|
swap in a new one (three adapter tests, and the health-check reconnect test,
|
||||||
|
now asserting it never reconnects).
|
||||||
|
- `sslmode` default `prefer`; `NewConnectionFromDB` connections are no longer
|
||||||
|
closed by the manager.
|
||||||
|
|
||||||
|
**Regression tests added:** `lifecycle_test.go` (double Connect/Close cycle,
|
||||||
|
idempotent Connect, concurrent Reconnect, adapter factory leaves pool open,
|
||||||
|
accessors not blocked by health check, Close marks disconnected, existing-DB
|
||||||
|
Reconnect/Close leave the caller's pool open), `config_dsn_test.go`,
|
||||||
|
`providers/pgconnector_test.go`, `pg_live_test.go` (refresh keeps handles,
|
||||||
|
listener Listen/Notify) and `restart_live_test.go` (server crash and restart).
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Idle-connection failure chain
|
||||||
|
|
||||||
|
This is how findings 1–5 combine into "the connection sat idle and then could
|
||||||
|
not be used":
|
||||||
|
|
||||||
|
1. The app is idle. A NAT, firewall, load balancer or pgbouncer silently drops
|
||||||
|
the idle TCP flows. No FIN or RST reaches the process.
|
||||||
|
2. The next request takes a pooled connection. pgx's `ResetSession` pings it
|
||||||
|
because it has been idle for more than 1 s, and that ping uses the request
|
||||||
|
ctx, **which has no deadline** (finding 4). The write goes into the kernel
|
||||||
|
buffer and the read blocks until TCP retransmission gives up, which can
|
||||||
|
take minutes.
|
||||||
|
Meanwhile the health checker's 5 s ping times out and holds `c.mu`
|
||||||
|
**exclusively** for the whole time (finding 3), so every request trying to
|
||||||
|
get a handle queues behind it.
|
||||||
|
3. Eventually something returns "sql: database is closed" or
|
||||||
|
`ErrConnectionClosed`. That can be an adapter that hit a closed pool, or a
|
||||||
|
partial `Close` (finding 9). An adapter's `dbFactory` or the health checker
|
||||||
|
then calls `Reconnect` (finding 2).
|
||||||
|
4. `Reconnect` closes the `*sql.DB` (finding 1). If the Postgres listener has
|
||||||
|
subscriptions, `Close` first sends `UNLISTEN` on its own dead socket with no
|
||||||
|
deadline, still holding the write lock (finding 5), which freezes the
|
||||||
|
process again.
|
||||||
|
5. When the reconnect completes, every handle captured before it is
|
||||||
|
permanently broken. That includes the `*gorm.DB` given to
|
||||||
|
`resolvespec.NewHandlerWithGORM` in `cmd/testserver/main.go:142,56`, any
|
||||||
|
`*bun.DB` passed to `NewHandlerWithBun`, and every Bun `NewSelect`/`NewInsert`
|
||||||
|
path. **From this point on, every request that goes through those handles
|
||||||
|
fails until the process is restarted.** Concurrent failures run their own
|
||||||
|
`Reconnect`s, and each one closes the pool the previous one just opened
|
||||||
|
(finding 2).
|
||||||
|
|
||||||
|
### Fix order for this symptom
|
||||||
|
|
||||||
|
1. **Stop closing the pool to recover from connection errors.** Remove
|
||||||
|
`WithDBFactory(c.reopen*ForAdapter)` → `Reconnect`, and remove the
|
||||||
|
health-check → `Reconnect` path for SQL providers. `*sql.DB` already discards
|
||||||
|
bad connections (`driver.ErrBadConn`, `ResetSession`,
|
||||||
|
`SetConnMaxIdleTime`/`SetConnMaxLifetime`). Keep `Reconnect` for explicit
|
||||||
|
operator use only, and make it atomic (finding 2).
|
||||||
|
2. Give every request a deadline. Wrap the request ctx in
|
||||||
|
`context.WithTimeout(ctx, QueryTimeout)` in the adapters, or at the handler
|
||||||
|
boundary.
|
||||||
|
3. Set `SetConnMaxIdleTime` **below** the shortest idle timeout of any
|
||||||
|
middlebox (typically 60–240 s for cloud NATs and LBs) so idle connections
|
||||||
|
are recycled before they can be dropped silently. Also set TCP keepalive and
|
||||||
|
`TCP_USER_TIMEOUT` through a custom `pgconn.Config.DialFunc`.
|
||||||
|
4. Ping without the write lock (finding 3), and give the listener's `Close`
|
||||||
|
bounded ctxs (finding 5).
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### 1. Critical — `Reconnect` kills every previously issued handle
|
||||||
|
|
||||||
|
`connection.go:129-160` (`Close`) and `connection.go:187-192` (`Reconnect`):
|
||||||
|
|
||||||
|
```go
|
||||||
|
func (c *sqlConnection) Close() error {
|
||||||
|
c.mu.Lock()
|
||||||
|
...
|
||||||
|
if c.bunDB != nil {
|
||||||
|
if err := c.bunDB.Close(); err != nil { // closes the shared *sql.DB
|
||||||
|
...
|
||||||
|
if err := c.provider.Close(); err != nil { // closes it again (idempotent)
|
||||||
|
...
|
||||||
|
c.nativeDB = nil
|
||||||
|
c.bunDB = nil
|
||||||
|
c.gormDB = nil
|
||||||
|
c.bunAdapter = nil
|
||||||
|
...
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *sqlConnection) Reconnect(ctx context.Context) error {
|
||||||
|
if err := c.Close(); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return c.Connect(ctx)
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
`Bun()`, `GORM()` and `Native()` return the handle itself, and callers keep
|
||||||
|
it: every spec package has a `NewHandlerWithGORM(*gorm.DB)` /
|
||||||
|
`NewHandlerWithBun(*bun.DB)` constructor, and `cmd/testserver/main.go:142` does
|
||||||
|
exactly this. After `Reconnect`, the cached fields are nilled, a new pool is
|
||||||
|
built, and the handles the callers hold point at a `*sql.DB` whose `closed`
|
||||||
|
flag is set forever.
|
||||||
|
|
||||||
|
Verified with a probe: I obtained `conn.GORM()`, called `conn.Reconnect(ctx)`,
|
||||||
|
then ran a query through the old handle. It returned
|
||||||
|
`sql: database is closed`, and a fresh `conn.GORM()` worked.
|
||||||
|
|
||||||
|
The comment in `manager.go:371-374` shows the authors already knew about this
|
||||||
|
("forcing Close()+Connect() here invalidates any cached ORM wrappers and callers
|
||||||
|
that still hold the old handle"). Their mitigation was to narrow *when* the
|
||||||
|
health checker reconnects. But the adapters' own `dbFactory` still reconnects
|
||||||
|
unconditionally (finding 2).
|
||||||
|
|
||||||
|
**Failure scenario.** Any event that triggers a reconnect turns every
|
||||||
|
long-lived handler into a permanent 500 generator: a single adapter query hitting
|
||||||
|
"database is closed", or a health check returning `ErrConnectionClosed`. The
|
||||||
|
process does not recover without a restart. The same thing happens after a
|
||||||
|
normal `Manager.Close()` + `Connect()` in tests or hot-reload code.
|
||||||
|
|
||||||
|
**Recommendation.** Treat the `*sql.DB` as immortal for the life of the
|
||||||
|
`sqlConnection`. Don't close it to "reconnect": `database/sql` already replaces
|
||||||
|
broken connections. If a real re-dial is ever needed (for example after
|
||||||
|
changing credentials), build the new pool, atomically swap it in, and close the
|
||||||
|
old one only after a grace period. Give the handles returned by
|
||||||
|
`Bun()`/`GORM()`/`Native()` stable identity; one way is a `driver.Connector`
|
||||||
|
that indirects to the current pool.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### 2. High — Adapter-triggered, non-atomic `Reconnect` causes a reconnect storm
|
||||||
|
|
||||||
|
`connection.go:362-397` and `connection.go:431/474/517-525`:
|
||||||
|
|
||||||
|
```go
|
||||||
|
func (c *sqlConnection) reconnectForAdapter() error {
|
||||||
|
...
|
||||||
|
return c.Reconnect(ctx) // Close() then Connect(): two separate lock scopes
|
||||||
|
}
|
||||||
|
...
|
||||||
|
WithDBFactory(c.reopenBunForAdapter).
|
||||||
|
```
|
||||||
|
|
||||||
|
The adapters (`pkg/common/adapters/database/bun.go:131`, `gorm.go`,
|
||||||
|
`pgsql.go`) call `dbFactory` whenever an operation returns an error that
|
||||||
|
matches `"sql: database is closed"`. So:
|
||||||
|
|
||||||
|
- **One stale handle closes the pool for everyone.** If an adapter holds a
|
||||||
|
`*sql.DB` from before a previous reconnect, its first query fails with
|
||||||
|
"database is closed". Its factory then calls `c.Reconnect`, which closes the
|
||||||
|
*current, healthy* pool that every other adapter and request is using right
|
||||||
|
now.
|
||||||
|
- **`Reconnect` isn't atomic.** `Close` and `Connect` each take `c.mu`
|
||||||
|
separately. Under N concurrent failures, one goroutine closes and reconnects
|
||||||
|
while the others either close the brand-new pool again or fail with
|
||||||
|
`already connected`. The probe used 20 concurrent `Reconnect`s: 9 returned
|
||||||
|
"already connected", and every successful reconnect closed the pool the
|
||||||
|
previous winner had just handed to its adapter. Each of those adapters then
|
||||||
|
sees "database is closed" on its next query, and the cycle continues.
|
||||||
|
|
||||||
|
**Failure scenario.** A burst of traffic arrives just after a reconnect. Each
|
||||||
|
in-flight request whose adapter still holds the old pool triggers another
|
||||||
|
`Reconnect`, and each of those closes the pool that the previous request
|
||||||
|
reopened. The service flaps until traffic stops.
|
||||||
|
|
||||||
|
**Recommendation.** Remove the adapter → `Reconnect` path (see finding 1). If
|
||||||
|
it is kept, make `Reconnect` a single critical section, and add a generation
|
||||||
|
counter: a caller that saw generation N only reconnects if the current
|
||||||
|
generation is still N; otherwise it just re-fetches the handle.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### 3. High — Health check holds the write lock across a network ping
|
||||||
|
|
||||||
|
`connection.go:163-185`:
|
||||||
|
|
||||||
|
```go
|
||||||
|
func (c *sqlConnection) HealthCheck(ctx context.Context) error {
|
||||||
|
c.mu.Lock() // exclusive
|
||||||
|
defer c.mu.Unlock()
|
||||||
|
...
|
||||||
|
if err := c.provider.HealthCheck(ctx); err != nil { // PingContext, 5 s timeout
|
||||||
|
```
|
||||||
|
|
||||||
|
Every handle accessor takes `c.mu.RLock()` first (`connection.go:199, 238, 271,
|
||||||
|
308, 335, 403, 441, 484`). While the health checker (every 15 s, `manager.go:348`)
|
||||||
|
is pinging, **every request that needs a DB handle waits**. On a healthy
|
||||||
|
network this is a few ms. On a dead idle socket it's the full 5 s ping timeout
|
||||||
|
(`providers/postgres.go:155`, inside a 10 s outer ctx).
|
||||||
|
|
||||||
|
Verified with a probe: while `c.mu` was held, `conn.Bun()` blocked for the whole
|
||||||
|
hold (200 ms in the test).
|
||||||
|
|
||||||
|
**Failure scenario.** A network blip or a silently dropped idle connection
|
||||||
|
makes the ping hang. Every 15 s the whole API pauses for up to 5 s. This fits
|
||||||
|
reports of "idle, then slow or unusable".
|
||||||
|
|
||||||
|
**Recommendation.** Snapshot `provider` under `RLock`, release the lock, ping,
|
||||||
|
then take the lock only to write `healthCheckStatus` / `lastHealthCheck`. Better
|
||||||
|
still, keep the status in an `atomic.Value`.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### 4. High — No client-side query deadline; `QueryTimeout` is server-side only and floored at 2 min
|
||||||
|
|
||||||
|
`config.go:223-228`:
|
||||||
|
|
||||||
|
```go
|
||||||
|
if cc.QueryTimeout == 0 {
|
||||||
|
cc.QueryTimeout = 2 * time.Minute
|
||||||
|
} else if cc.QueryTimeout < 2*time.Minute {
|
||||||
|
cc.QueryTimeout = 2 * time.Minute
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
`config.go:331-335` turns this into `statement_timeout=<ms>` in the Postgres DSN,
|
||||||
|
and it only does that when the DSN is *built*. A user-supplied `DSN` gets no
|
||||||
|
timeout at all. Nothing anywhere in the request path wraps ctx in a deadline.
|
||||||
|
`pkg/config`'s `query_timeout: 30s` default is silently raised to 2 min.
|
||||||
|
|
||||||
|
`statement_timeout` is enforced by the **server**, so it only helps if the
|
||||||
|
server is reachable. On a silently dropped connection:
|
||||||
|
|
||||||
|
- pgconn's default dialer is `&net.Dialer{}`: Go's default keepalive (15 s idle,
|
||||||
|
15 s interval, 9 probes) and **no `TCP_USER_TIMEOUT`**.
|
||||||
|
- Once a query has been written, there is unacknowledged data, so keepalive does
|
||||||
|
not apply. The socket then waits for TCP retransmission to give up
|
||||||
|
(`tcp_retries2`), which takes about 15 min on Linux defaults.
|
||||||
|
- `database/sql` calls pgx's `ResetSession`, which pings a connection that has
|
||||||
|
been idle for more than 1 s. That ping uses the **request ctx**, so with no
|
||||||
|
deadline it blocks just as long.
|
||||||
|
|
||||||
|
**Failure scenario.** An idle period longer than the NAT or LB idle timeout
|
||||||
|
causes the next request to hang for minutes rather than failing fast and being
|
||||||
|
retried on a fresh connection. With `MaxOpenConns` = 25, 25 such requests
|
||||||
|
exhaust the pool and every later request blocks on `db.conn()`.
|
||||||
|
|
||||||
|
**Recommendation.**
|
||||||
|
- Apply `context.WithTimeout(ctx, QueryTimeout)` in the adapters, or in a
|
||||||
|
handler middleware.
|
||||||
|
- Remove the 2-minute floor, and honour the configured value.
|
||||||
|
- Set `SetConnMaxIdleTime` below the middlebox idle timeout.
|
||||||
|
- Configure `pgconn.Config.DialFunc` with a `net.Dialer` that has `KeepAlive`
|
||||||
|
set and a `Control` func setting `TCP_USER_TIMEOUT` (for example 30 s).
|
||||||
|
- Apply `statement_timeout` through `RuntimeParams` so it also works with a
|
||||||
|
supplied DSN.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### 5. High — Listener `Close` does unbounded network I/O under three locks
|
||||||
|
|
||||||
|
`providers/postgres_listener.go:216-244`, reached from
|
||||||
|
`providers/postgres.go:116-126`, which is reached from `connection.go:147`:
|
||||||
|
|
||||||
|
```go
|
||||||
|
// sqlConnection.Close holds c.mu (write)
|
||||||
|
// PostgresProvider.Close holds p.mu
|
||||||
|
// PostgresListener.Close holds l.mu:
|
||||||
|
for channel := range l.channels {
|
||||||
|
_, _ = l.conn.Exec(context.Background(), fmt.Sprintf("UNLISTEN %s", ...))
|
||||||
|
}
|
||||||
|
err := l.conn.Close(context.Background())
|
||||||
|
```
|
||||||
|
|
||||||
|
If the listener's socket is dead, and it usually is in the situation that
|
||||||
|
triggers a reconnect, each `UNLISTEN` waits for a reply that never comes. This
|
||||||
|
is the same unbounded wait as in finding 4, and `c.mu` is held **for writing**
|
||||||
|
the whole time. Every request blocks. `bunDB` has already been closed at this
|
||||||
|
point, so there is no fallback either.
|
||||||
|
|
||||||
|
Also, if `listener.Close` returns an error, `PostgresProvider.Close` returns
|
||||||
|
early. `sqlConnection.Close` then returns with `connected=true` over a closed
|
||||||
|
pool (finding 9).
|
||||||
|
|
||||||
|
**Failure scenario.** An app with any `LISTEN` subscription hits a network
|
||||||
|
partition. The health checker or an adapter calls `Reconnect`, and the process
|
||||||
|
stops serving database requests for as long as the kernel takes to kill the
|
||||||
|
socket.
|
||||||
|
|
||||||
|
**Recommendation.** Skip `UNLISTEN` entirely, because closing the connection
|
||||||
|
drops all subscriptions server-side. Close with `context.WithTimeout(…, 2*time.Second)`.
|
||||||
|
Don't do network I/O while holding `l.mu`, and don't close the listener
|
||||||
|
inside `sqlConnection.Close`'s write lock.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### 6. High — Listener leaks a goroutine pair per reconnect, and they race on one `pgx.Conn`
|
||||||
|
|
||||||
|
`providers/postgres_listener.go:48-120` (Connect), `257-324` (handleNotifications),
|
||||||
|
`326-370` (handleReconnection).
|
||||||
|
|
||||||
|
`Connect()` ends by starting `go l.handleNotifications()` and
|
||||||
|
`go l.handleReconnection()`. `handleReconnection` responds to a reconnect
|
||||||
|
signal by calling `l.Connect(ctx)`, which starts **another** pair. The old pair
|
||||||
|
keeps running on the same `l.ctx`. After N reconnects there are N+1
|
||||||
|
notification loops. Each one snapshots `l.conn` and calls
|
||||||
|
`conn.WaitForNotification`. `pgx.Conn` is **not** safe for concurrent use, so
|
||||||
|
the second caller gets a "conn busy" error. That error isn't a timeout, so it
|
||||||
|
sends another reconnect signal, which adds another pair.
|
||||||
|
|
||||||
|
`handleReconnection` also waits with `time.Sleep(5 * time.Second)` instead of
|
||||||
|
selecting on `l.ctx.Done()`, so `Close` can't interrupt it. And `Listen` runs
|
||||||
|
`l.conn.Exec(LISTEN …)` while holding `l.mu`, which blocks `handleReconnection`
|
||||||
|
for as long as that Exec takes.
|
||||||
|
|
||||||
|
Once the parent `PostgresProvider` is closed (for example by any `Reconnect`,
|
||||||
|
finding 1), subscribers holding the old `*PostgresListener` get
|
||||||
|
"listener is closed" forever. Nothing re-subscribes them on the new provider.
|
||||||
|
|
||||||
|
**Failure scenario.** A flaky network causes a few listener reconnects. The
|
||||||
|
goroutine count grows without bound, notifications are delivered twice or
|
||||||
|
dropped, and CPU rises because of the busy/reconnect spiral.
|
||||||
|
|
||||||
|
**Recommendation.** Start the goroutines once, in the constructor or the first
|
||||||
|
`Connect`. Have `handleReconnection` dial a new conn without calling the public
|
||||||
|
`Connect`. Guard `WaitForNotification` so only one loop owns the conn. Replace
|
||||||
|
`time.Sleep` with `select { case <-time.After(d): case <-l.ctx.Done(): }`.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### 7. High — Second `Close` panics; health checker silently dead after first cycle
|
||||||
|
|
||||||
|
`manager.go:119, 313-345`:
|
||||||
|
|
||||||
|
```go
|
||||||
|
stopChan: make(chan struct{}), // created once, in the constructor
|
||||||
|
...
|
||||||
|
func (m *connectionManager) stopHealthChecker() {
|
||||||
|
if m.healthTicker != nil {
|
||||||
|
m.healthTicker.Stop()
|
||||||
|
close(m.stopChan) // never recreated
|
||||||
|
m.wg.Wait()
|
||||||
|
m.healthTicker = nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
After `Connect → Close`, `stopChan` is closed. A second `Connect` calls
|
||||||
|
`startHealthChecker`, which creates a new ticker and goroutine. That goroutine's
|
||||||
|
`select` sees the closed `stopChan` right away and **exits**, so health
|
||||||
|
checking is silently off. A second `Close` finds `healthTicker != nil` and
|
||||||
|
calls `close(m.stopChan)` again, which **panics**: `close of closed channel`.
|
||||||
|
`startHealthChecker` and `stopHealthChecker` also read and write `healthTicker`
|
||||||
|
without `m.mu` held (`Close` calls `stopHealthChecker` before locking), so a
|
||||||
|
concurrent `Connect`/`Close` pair is a data race.
|
||||||
|
|
||||||
|
Calling `Connect` twice without `Close` also leaks: `m.connections[name] = conn`
|
||||||
|
overwrites the previous connection without closing it.
|
||||||
|
|
||||||
|
**Failure scenario.** Anything that cycles the manager can crash the process
|
||||||
|
during shutdown: graceful restart, config hot-reload, or test suites using
|
||||||
|
`ResetInstance`.
|
||||||
|
|
||||||
|
**Recommendation.** Create `stopChan` in `startHealthChecker`. Guard both
|
||||||
|
functions with `m.mu`, or a dedicated mutex. Make `Connect` idempotent, or have
|
||||||
|
it close existing connections first.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### 8. Medium — SQLite: in-memory data loss and per-connection pragmas
|
||||||
|
|
||||||
|
`providers/sqlite.go:54-90`, `config.go:140-141, 202-204`:
|
||||||
|
|
||||||
|
- `ManagerConfig.ApplyDefaults` always gives `MaxOpenConns` a value (25), so the
|
||||||
|
"SQLite works best with MaxOpenConns=1" branch at `sqlite.go:60` never runs.
|
||||||
|
The probe reported `MaxOpenConnections=25`.
|
||||||
|
- With `:memory:` (the documented test setup), each pooled connection opens its
|
||||||
|
**own** private database. The probe created a table on one connection, and a
|
||||||
|
second connection reported `no such table: t`. `ConnMaxIdleTime` (default
|
||||||
|
5 min) then closes idle connections and their data with them.
|
||||||
|
- `PRAGMA journal_mode=WAL` and `PRAGMA busy_timeout` are `Exec`'d once on
|
||||||
|
whichever pooled connection runs them. `busy_timeout` is per-connection, so
|
||||||
|
the other 24 get `database is locked` immediately under write contention.
|
||||||
|
- `BuildDSN` adds `?_timeout=<ms>` (`config.go:347-351`), but
|
||||||
|
`glebarez/go-sqlite` only recognises `_pragma`, `_txlock` and `_time_format`,
|
||||||
|
so this parameter is silently ignored.
|
||||||
|
- `SQLiteProvider.reconnectDB` (`sqlite.go:165`) needs a `dbFactory` that
|
||||||
|
nothing ever sets, so it is dead code.
|
||||||
|
|
||||||
|
**Recommendation.** For SQLite, force `MaxOpenConns=1` for `:memory:` (or use
|
||||||
|
`file::memory:?cache=shared`), and never set an idle timeout there. Pass the
|
||||||
|
pragmas in the DSN (`_pragma=busy_timeout(5000)&_pragma=journal_mode(WAL)`) so
|
||||||
|
every connection gets them.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### 9. Medium — Partial-failure states in `Close` and `Connect`
|
||||||
|
|
||||||
|
- `connection.go:137-149`: if `bunDB.Close()` or `provider.Close()` fails, for
|
||||||
|
example because the listener's Close failed (finding 5), `Close` returns
|
||||||
|
early with `connected = true` and the pool already closed. Every accessor then
|
||||||
|
returns a handle to a closed pool until someone calls `Close` again.
|
||||||
|
- `manager.go:197-231`: if connection *k* of *n* fails to connect, `Connect`
|
||||||
|
returns an error. Connections 1…k-1 stay open but are never stored in
|
||||||
|
`m.connections`, so `Close` can't reach them and they leak.
|
||||||
|
|
||||||
|
**Recommendation.** In `Close`, mark the connection disconnected and nil the
|
||||||
|
fields regardless of errors, and return a joined error. In `Connect`, close any
|
||||||
|
connections opened so far when a later one fails.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### 10. Medium — DSN builders don't escape credentials; TLS off by default
|
||||||
|
|
||||||
|
`config.go` `buildPostgresDSN` / `buildMSSQLDSN` / `buildMongoDSN` use
|
||||||
|
`fmt.Sprintf` with raw `User`/`Password`/`Database` values:
|
||||||
|
|
||||||
|
- Postgres key=value format: a password containing a space or `'` breaks
|
||||||
|
parsing. A password like `x sslmode=disable` *overrides earlier parameters*.
|
||||||
|
- MSSQL and Mongo URLs: `@`, `:`, `/`, `?` or `&` in the password corrupt the
|
||||||
|
URL. They need `url.QueryEscape` / `url.UserPassword`.
|
||||||
|
- `sslmode` defaults to `disable` (`config.go:322-325`); see
|
||||||
|
`_CROSS-CUTTING.audit.md` X6.
|
||||||
|
|
||||||
|
These values come from config, not from clients, so this isn't directly
|
||||||
|
exploitable by the threat model. It is a correctness and hardening problem,
|
||||||
|
and it becomes a security problem wherever DSN parts come from a tenant or
|
||||||
|
operator UI.
|
||||||
|
|
||||||
|
**Recommendation.** Build the Postgres DSN as a URL with `url.URL{User: url.UserPassword(...)}`,
|
||||||
|
or quote key=value values properly. Default `sslmode` to `prefer` or `require`.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### 11. Medium — Config knobs that are ignored or cannot be disabled
|
||||||
|
|
||||||
|
- `config.go:161-168`: `HealthCheckInterval == 0` and
|
||||||
|
`EnableAutoReconnect == false` are both treated as "unset" and replaced with
|
||||||
|
the defaults (15 s, `true`). **Auto-reconnect, the trigger for findings 1–2,
|
||||||
|
cannot be switched off from config.**
|
||||||
|
- `RetryAttempts`, `RetryDelay` and `RetryMaxDelay` are defaulted and copied,
|
||||||
|
but no provider reads them. Every provider hardcodes `retryAttempts := 3`
|
||||||
|
and `retryDelay := 1 * time.Second`.
|
||||||
|
- `statement_timeout` is only added when the DSN is built (finding 4), and
|
||||||
|
SQLite `_timeout` is ignored by the driver (finding 8).
|
||||||
|
|
||||||
|
**Recommendation.** Use `*bool` / `*time.Duration`, or an explicit
|
||||||
|
`Disable…` flag, for the values that can legitimately be zero or false. Wire
|
||||||
|
the retry settings into the providers, or delete them.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### 12. Medium — `Manager.Connect` holds the manager lock across network dials
|
||||||
|
|
||||||
|
`manager.go:197-231` holds `m.mu` (write) while dialing every configured
|
||||||
|
connection, each with up to 3 attempts, backoff, and `ConnectTimeout`.
|
||||||
|
`GetConnection`, `HealthCheck`, `Stats` and the health checker all wait
|
||||||
|
behind it. That's harmless at startup, but it serialises the whole manager if
|
||||||
|
`Connect` is ever called at runtime (hot-reload, lazy init).
|
||||||
|
|
||||||
|
**Recommendation.** Dial outside the lock, then lock only to publish the
|
||||||
|
results into `m.connections`.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### 13. Low — dbmanager metrics are never published
|
||||||
|
|
||||||
|
`metrics.go` defines Prometheus collectors plus `PublishMetrics` and
|
||||||
|
`RecordReconnectAttempt`. A grep over the repository finds **no callers** of
|
||||||
|
either. The connection-pool gauges (open, in-use, idle, wait count) are exactly
|
||||||
|
what would have shown the idle-connection problem, and they are always zero.
|
||||||
|
The `*_total` names are registered as gauges, not counters.
|
||||||
|
|
||||||
|
**Recommendation.** Call `PublishMetrics` from the health-check tick, call
|
||||||
|
`RecordReconnectAttempt` from `Reconnect`, and make the totals counters.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### 14. Low — Assorted correctness issues
|
||||||
|
|
||||||
|
- `Native()` checks `c.connected` (`connection.go:214`); `Bun()` and `GORM()`
|
||||||
|
don't. After a partial `Close` they can build ORM wrappers over a nil or
|
||||||
|
closed DB.
|
||||||
|
- `getNativeAdapter` (`connection.go:500-525`) wraps SQLite and MSSQL in
|
||||||
|
`PgSQLAdapter`, which quotes and builds SQL in Postgres dialect.
|
||||||
|
- `ExistingDBProvider` (`NewConnectionFromDB`) applies no pool settings and no
|
||||||
|
idle or lifetime limits, and its `Close` closes the caller's `*sql.DB`.
|
||||||
|
- `MongoProvider` uses `MaxIdleConns` as `MinPoolSize`, and `Stats()` returns an
|
||||||
|
empty struct.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### 15. Low — Logging defects
|
||||||
|
|
||||||
|
- `manager.go:247, 367-369, 378-380` call `logger.Error("…", "name", name, "error", err)`.
|
||||||
|
`pkg/logger` is printf-style, so these print `%!(EXTRA string=name, …)`, and
|
||||||
|
the error text is buried in exactly the log lines needed during an outage.
|
||||||
|
- `ResetInstance` discards the error from `Close`.
|
||||||
|
- Connection errors wrap driver errors that can include the DSN host and user.
|
||||||
|
Together with `_CROSS-CUTTING.audit.md` X8, they reach Sentry unscrubbed.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Test coverage
|
||||||
|
|
||||||
|
`manager_test.go` and `factory_test.go` cover construction and config defaults.
|
||||||
|
Nothing tests `Reconnect` while handles are held, concurrent `Reconnect`, a
|
||||||
|
`Connect`/`Close` cycle run twice, health-check lock hold time, or listener
|
||||||
|
reconnection. Each of findings 1, 2, 3, 6 and 7 can be reproduced with a short
|
||||||
|
SQLite-backed test (the probes used for this audit took about 20 lines each).
|
||||||
|
Add them as regression tests when the fixes land, and run them with `-race`
|
||||||
|
(`_CROSS-CUTTING.audit.md` X1).
|
||||||
@@ -0,0 +1,220 @@
|
|||||||
|
# Audit — `pkg/errortracking`
|
||||||
|
|
||||||
|
- **Date:** 2026-09-29
|
||||||
|
- **Scope:** `pkg/errortracking/{interfaces,noop,sentry,factory}.go` (260 LOC, 4 source files + 1 test file, 67 LOC)
|
||||||
|
- **Axes:** thread locking/waiting · slowness · security · panic handling & logging
|
||||||
|
- **Threat model:** hostile internet client; error messages and `extra` maps may contain attacker-shaped content.
|
||||||
|
|
||||||
|
## Summary
|
||||||
|
|
||||||
|
Small, clean abstraction: a `Provider` interface, a no-op implementation, a Sentry implementation,
|
||||||
|
and a config-driven factory. The concurrency story is fine — `sentry.Hub` is internally
|
||||||
|
mutex-guarded and the provider holds no mutable state of its own. The real exposure is **what
|
||||||
|
this package sends out of the trust boundary**: it is the egress point for every `Warn`/`Error`
|
||||||
|
in the codebase (see `audit/pkg/logger.audit.md` findings 2 and 3) and it applies **no scrubbing
|
||||||
|
whatsoever**.
|
||||||
|
|
||||||
|
| # | Severity | Axis | Finding |
|
||||||
|
|---|----------|------|---------|
|
||||||
|
| 1 | **High** | Security | No `BeforeSend` scrubber — messages, stack traces and `extra` leave the trust boundary verbatim |
|
||||||
|
| 2 | Medium | Security | `sentry.Init` mutates process-global state; `NewSentryProvider` can be called repeatedly and silently replaces the global client |
|
||||||
|
| 3 | Medium | Slowness | `Flush(timeout int)` is second-granularity only; combined with `Close()` gives up to 7 s of shutdown stall |
|
||||||
|
| 4 | Medium | Slowness | `CapturePanic` stringifies the whole stack trace into an `extra` field on every panic |
|
||||||
|
| 5 | Low | Security | `AttachStacktrace: true` is hardcoded — source paths and function names of the deployment leak to the SaaS |
|
||||||
|
| 6 | Low | Correctness | `CaptureError` produces an `Exception` with a nil `Stacktrace` for plain `errors.New` values |
|
||||||
|
| 7 | Low | Correctness | Config-provided `SampleRate == 0` silently means "send everything", not "send nothing" |
|
||||||
|
| 8 | Low | Architecture | `factory.go` imports `pkg/config`, coupling the lowest-level package to the config layer |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Findings
|
||||||
|
|
||||||
|
### 1. No scrubbing before egress (High, Security)
|
||||||
|
|
||||||
|
`sentry.go:29-42`
|
||||||
|
|
||||||
|
```go
|
||||||
|
err := sentry.Init(sentry.ClientOptions{
|
||||||
|
Dsn: config.DSN,
|
||||||
|
Environment: config.Environment,
|
||||||
|
Release: config.Release,
|
||||||
|
Debug: config.Debug,
|
||||||
|
AttachStacktrace: true,
|
||||||
|
SampleRate: config.SampleRate,
|
||||||
|
TracesSampleRate: config.TracesSampleRate,
|
||||||
|
})
|
||||||
|
```
|
||||||
|
|
||||||
|
`BeforeSend` is not set. Neither is `BeforeSendTransaction`. Nothing in `CaptureError`
|
||||||
|
(`sentry.go:46`), `CaptureMessage` (`sentry.go:75`) or `CapturePanic` (`sentry.go:97`) inspects or
|
||||||
|
redacts its inputs; all three copy straight into `event.Message` / `event.Exception.Value` /
|
||||||
|
`event.Contexts["extra"]` and hand it to `hub.CaptureEvent`.
|
||||||
|
|
||||||
|
Because `pkg/logger.Error`/`Warn` forward every formatted message here unconditionally, the set of
|
||||||
|
things that can reach Sentry is "every error string produced anywhere in ResolveSpec". In this
|
||||||
|
codebase that includes driver errors (which embed DSNs and sometimes credentials on connect
|
||||||
|
failure), SQL fragments with bound values, and identifiers taken from request headers.
|
||||||
|
|
||||||
|
Under the hostile-client threat model this is an **attacker-reachable exfiltration channel**: shape
|
||||||
|
an input that lands in an error message, and its content is written to a third-party system
|
||||||
|
outside the operator's control.
|
||||||
|
|
||||||
|
**Recommendation:** set `BeforeSend` to run a redaction pass over `Message`,
|
||||||
|
`Exception[].Value` and `Contexts` — at minimum strip `password=`, `://user:pass@`, `Bearer `,
|
||||||
|
and anything matching the configured DSN patterns. Consider an `extra`-key allowlist rather than
|
||||||
|
passing the caller's map through (`sentry.go:70`, `92`, `114-121`).
|
||||||
|
|
||||||
|
### 2. `sentry.Init` mutates process-global state (Medium, Security/Correctness)
|
||||||
|
|
||||||
|
`sentry.go:29` calls the package-level `sentry.Init`, which installs a global client, and
|
||||||
|
`sentry.go:40` then captures `sentry.CurrentHub()`. Consequences:
|
||||||
|
|
||||||
|
- Calling `NewSentryProvider` twice (two `NewProviderFromConfig` calls, or a config reload)
|
||||||
|
replaces the global client. Any previously-created `SentryProvider` keeps a `hub` pointer whose
|
||||||
|
client has been swapped underneath it — events start going to the *new* DSN. If the two configs
|
||||||
|
have different environments or DSNs, events are misrouted with no error.
|
||||||
|
- Events enqueued on the old client at swap time may be dropped without flush.
|
||||||
|
- It means this "provider" abstraction is a lie: you cannot actually have two Sentry providers
|
||||||
|
with different configs in one process.
|
||||||
|
|
||||||
|
**Recommendation:** build a dedicated client with `sentry.NewClient(opts)` and bind it to an
|
||||||
|
owned `sentry.NewHub(client, scope)` rather than touching the global. That also makes `Close()`
|
||||||
|
able to genuinely release resources.
|
||||||
|
|
||||||
|
### 3. Coarse, additive shutdown flush (Medium, Slowness)
|
||||||
|
|
||||||
|
`sentry.go:125-128`
|
||||||
|
|
||||||
|
```go
|
||||||
|
func (s *SentryProvider) Flush(timeout int) bool {
|
||||||
|
return sentry.Flush(time.Duration(timeout) * time.Second)
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
`timeout` is an `int` interpreted as whole seconds — the interface (`interfaces.go:30`) cannot
|
||||||
|
express 500 ms. `Close()` (`sentry.go:131-134`) then runs a *second* `sentry.Flush(2s)`.
|
||||||
|
|
||||||
|
`pkg/logger.CloseErrorTracking` (`logger.go:69-75`) calls `Flush(5)` then `Close()`, so a graceful
|
||||||
|
shutdown blocks for **up to 7 seconds** in this package alone, before the HTTP drain and DB close
|
||||||
|
budgets in `pkg/server`. If the Sentry endpoint is unreachable (the common case during an
|
||||||
|
outage — which is when you are restarting) both flushes run to full timeout.
|
||||||
|
|
||||||
|
Note `Flush` also flushes the *global* client, not `s.hub`'s, which is the same object today only
|
||||||
|
because of finding 2.
|
||||||
|
|
||||||
|
**Recommendation:** change the interface to `Flush(context.Context) bool` or
|
||||||
|
`Flush(time.Duration) bool`; have `Close` not re-flush; and pass the server's shutdown deadline
|
||||||
|
through instead of hardcoding 5.
|
||||||
|
|
||||||
|
### 4. Whole stack trace stringified into `extra` on every panic (Medium, Slowness)
|
||||||
|
|
||||||
|
`sentry.go:117-119`
|
||||||
|
|
||||||
|
```go
|
||||||
|
if stackTrace != nil {
|
||||||
|
extraCtx["stack_trace"] = string(stackTrace)
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
The caller (`pkg/logger.CatchPanicCallback`, `HandlePanic`) already produced the trace via
|
||||||
|
`debug.Stack()`. Here it is copied again into a string and shipped as a context field. Per
|
||||||
|
recovered panic that's two full copies of a multi-kilobyte trace plus a network event. With
|
||||||
|
panics recovered rather than fatal on the request path, a reliably-panicking input is a cheap
|
||||||
|
amplification primitive (see `audit/pkg/logger.audit.md` finding 5).
|
||||||
|
|
||||||
|
Sentry also truncates large context values server-side, so much of this payload is wasted.
|
||||||
|
|
||||||
|
**Recommendation:** put the trace in `Exception[0].Stacktrace` as structured frames (which Sentry
|
||||||
|
groups and displays properly) rather than a blob in `extra`, and cap the byte length.
|
||||||
|
|
||||||
|
### 5. `AttachStacktrace: true` hardcoded (Low, Security)
|
||||||
|
|
||||||
|
`sentry.go:35`. Not configurable. Every event carries absolute source paths, package layout and
|
||||||
|
function names of the build. That's mostly a reconnaissance leak to whoever can read the Sentry
|
||||||
|
project rather than to the internet attacker, but it should be an operator choice, especially for
|
||||||
|
on-prem deployments sending to a hosted DSN.
|
||||||
|
|
||||||
|
### 6. Nil stack trace for plain errors (Low, Correctness)
|
||||||
|
|
||||||
|
`sentry.go:62`
|
||||||
|
|
||||||
|
```go
|
||||||
|
Stacktrace: sentry.ExtractStacktrace(err),
|
||||||
|
```
|
||||||
|
|
||||||
|
`ExtractStacktrace` only finds a trace if the error implements `StackTrace()`/`Callers()`
|
||||||
|
(`pkg/errors`-style). Nearly all errors in this codebase come from `fmt.Errorf`, so this returns
|
||||||
|
`nil` and the Sentry event has an exception with no frames — grouping falls back to the message
|
||||||
|
string, which (because messages embed request-specific values) fragments what should be one issue
|
||||||
|
into thousands.
|
||||||
|
|
||||||
|
**Recommendation:** fall back to `sentry.NewStacktrace()` when extraction yields nil, and set an
|
||||||
|
explicit `event.Fingerprint` derived from a stable prefix rather than the full message.
|
||||||
|
|
||||||
|
### 7. `SampleRate == 0` means "send everything" (Low, Correctness)
|
||||||
|
|
||||||
|
`factory.go:20-27` passes `cfg.SampleRate` through untouched, and `pkg/config/manager.go`
|
||||||
|
registers **no default** for `error_tracking.sample_rate`. So an operator who leaves it out gets
|
||||||
|
`0.0`, and `sentry-go@v0.46.2` `client.go:339-341` rewrites `0.0` → `1.0`.
|
||||||
|
|
||||||
|
Verified in the module cache:
|
||||||
|
|
||||||
|
```go
|
||||||
|
if options.SampleRate == 0.0 {
|
||||||
|
options.SampleRate = 1.0
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
Fail-open rather than fail-closed, which is arguably the right choice for an error tracker — but
|
||||||
|
it means an operator who *intends* to disable sampling by setting `0` gets the opposite, silently.
|
||||||
|
|
||||||
|
**Recommendation:** make `SampleRate` a `*float64` in the config struct, or register an explicit
|
||||||
|
default in `setDefaults`, and validate/log the effective value at init.
|
||||||
|
|
||||||
|
### 8. `factory.go` imports `pkg/config` (Low, Architecture)
|
||||||
|
|
||||||
|
`factory.go:6` — `errortracking` is imported by `pkg/logger`, which is imported by essentially
|
||||||
|
everything. Pulling `pkg/config` (and therefore `viper`) into that dependency chain means the
|
||||||
|
lowest-level logging path transitively depends on the configuration layer. It works today only
|
||||||
|
because `pkg/config` imports nothing from ResolveSpec; the first time it wants to log, there is
|
||||||
|
an import cycle.
|
||||||
|
|
||||||
|
**Recommendation:** move `NewProviderFromConfig` into `pkg/config`-adjacent wiring code (or take
|
||||||
|
a small local options struct instead of `config.ErrorTrackingConfig`) so `errortracking` stays a
|
||||||
|
leaf.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## What looks right
|
||||||
|
|
||||||
|
- **Concurrency is genuinely fine.** `SentryProvider` holds only an immutable `*sentry.Hub`;
|
||||||
|
`sentry.Hub` guards its own state with a mutex, and `CaptureEvent` hands off to a background
|
||||||
|
worker with a bounded queue, so it does not block the caller and does not need a lock here.
|
||||||
|
- `GetHubFromContext(ctx)` with fallback to `s.hub` (`sentry.go:53-56`, `81-84`, `103-106`) is the
|
||||||
|
correct Sentry idiom and preserves per-request scope when middleware installs a hub.
|
||||||
|
- Nil-input guards on all three capture methods (`sentry.go:47`, `76`, `98`) — a nil error, empty
|
||||||
|
message or nil recovered value is dropped rather than producing a junk event.
|
||||||
|
- `event.Contexts` is safe to index: `sentry.NewEvent()` initialises the map, so
|
||||||
|
`event.Contexts["extra"] = ...` cannot nil-panic.
|
||||||
|
- `NoOpProvider` means a disabled tracker is always safe to call — no nil checks needed at call
|
||||||
|
sites beyond the one in `pkg/logger`.
|
||||||
|
- `factory.go:15-17` correctly refuses to start with `provider: sentry` and an empty DSN rather
|
||||||
|
than silently no-oping.
|
||||||
|
|
||||||
|
## Panic handling
|
||||||
|
|
||||||
|
The package neither panics nor recovers, which is correct for its role — it is the *sink* for
|
||||||
|
panic reports, not a place that should be generating them. The nil-guards in finding "what looks
|
||||||
|
right" cover the realistic nil-deref paths. One residual: `CapturePanic` ranges over `extra`
|
||||||
|
(`sentry.go:115`) without a nil check, which is safe in Go (ranging a nil map yields zero
|
||||||
|
iterations) — noted only to confirm it was checked.
|
||||||
|
|
||||||
|
## Suggested follow-up
|
||||||
|
|
||||||
|
1. Add `BeforeSend` redaction (finding 1). This is the highest-value single change in the package.
|
||||||
|
2. Stop using the global Sentry client (finding 2) — unblocks real multi-provider support and a
|
||||||
|
meaningful `Close()`.
|
||||||
|
3. Widen `Flush` to a duration/context (finding 3) and wire it to the server shutdown budget.
|
||||||
|
4. Add tests for the Sentry path. The existing test file covers only `NoOpProvider`, severity
|
||||||
|
string mapping and interface satisfaction — `SentryProvider`'s capture methods have no
|
||||||
|
coverage at all. `sentry-go` ships a test transport that makes this straightforward.
|
||||||
@@ -0,0 +1,228 @@
|
|||||||
|
# Audit: `pkg/funcspec`
|
||||||
|
|
||||||
|
| | |
|
||||||
|
|---|---|
|
||||||
|
| **Package** | `github.com/bitechdev/ResolveSpec/pkg/funcspec` |
|
||||||
|
| **Files** | `function_api.go` (1251), `parameters.go` (411), `hooks.go` (179), `hooks_example.go`, `security_adapter.go` (117) |
|
||||||
|
| **Tests** | `function_api_test.go` (1278), `hooks_test.go` (589), `parameters_test.go` (549) — 2 416 lines; `go test` and `go test -race` pass, 76.1 % statement coverage |
|
||||||
|
| **Audit date** | 2026-09-30 |
|
||||||
|
| **Axes** | thread locking/waiting, slowness, security, panic handling & logging |
|
||||||
|
| **Threat model** | hostile internet client; query string, headers and body are attacker-controlled |
|
||||||
|
| **Depth** | targeted (server-side request path; verified against source) |
|
||||||
|
|
||||||
|
## Summary
|
||||||
|
|
||||||
|
`funcspec` exposes app-defined SQL templates as endpoints. The template is
|
||||||
|
trusted; everything the client adds to it is not. The package builds SQL by
|
||||||
|
string manipulation and has two kinds of client-controlled SQL fragments
|
||||||
|
(`X-Custom-SQL-W`, `X-Custom-SQL-Or`, `sort`) that are guarded only by a keyword
|
||||||
|
denylist (`ValidSQL(..., "select")`, `function_api.go:951-980`). That is not an
|
||||||
|
injection boundary: the fragment lands inside a query that may already carry
|
||||||
|
tenant or auth predicates, and the OR path produces wrong precedence that
|
||||||
|
widens results (findings 1-3).
|
||||||
|
|
||||||
|
The auth integration is weaker than it looks. `RegisterSecurityHooks` is opt-in,
|
||||||
|
the anonymous default is `UserID 0`, and the auth hooks return an error *and*
|
||||||
|
set `Abort`, so `Execute` returns the error first and the handler answers
|
||||||
|
**400 `hook_error`**, not 401 (finding 6).
|
||||||
|
|
||||||
|
Error handling leaks: `sendError` returns the DB error text and the full SQL to
|
||||||
|
the client, and the panic recovery writes the panic value into the 500 body
|
||||||
|
(finding 5).
|
||||||
|
|
||||||
|
Resource limits are absent: default limit 100 000, no cap on `X-Limit`, no
|
||||||
|
LIMIT at all when counting is skipped, unbounded `[post_body]` read, a 15-minute
|
||||||
|
timeout, and a `COUNT(1)` over the full query on every list request (finding 8).
|
||||||
|
|
||||||
|
Positives: there are no data races under the existing tests; the `[variable]`
|
||||||
|
substitution is quote-context aware; `Content-Type` and transaction handling are
|
||||||
|
consistent; hooks run inside the transaction.
|
||||||
|
|
||||||
|
## Findings
|
||||||
|
|
||||||
|
| # | Severity | Axis | Finding |
|
||||||
|
|---|---|---|---|
|
||||||
|
| 1 | **High** | security | `X-Custom-SQL-W`, `X-Custom-SQL-Or` and `sort` are appended as raw SQL, protected only by a keyword denylist (`ValidSQL "select"`) |
|
||||||
|
| 2 | **High** | security | `sqlQryWhereOr` emits `a AND b OR (c)`; OR conditions escape the AND-ed predicates (auth/tenant filters) |
|
||||||
|
| 3 | **Medium** | security | `sqlQryWhere`/`sqlQryWhereOr` locate WHERE/GROUP BY/ORDER BY/LIMIT by substring on the lower-cased query, including inside literals, subqueries and CTEs |
|
||||||
|
| 4 | **Medium** | security | Unquoted string `X-FieldFilter` value in `ApplyFilters`; `SearchOps` keyed per column (one op per column, random order) |
|
||||||
|
| 5 | **High** | security / logging | `sendError` returns `err.Error()` and the full SQL; panic recovery writes the panic value to the 500 body |
|
||||||
|
| 6 | **High** | security / correctness | Auth hooks return an error plus `Abort`; handler replies 400 `hook_error` instead of 401; hooks are opt-in; anonymous = `UserID 0` |
|
||||||
|
| 7 | **Medium** | security | `DecodeParam` (`ZIP_`/`__`) is applied to every header/param, ignores errors and recurses without a depth limit; headers matched with `HasPrefix` |
|
||||||
|
| 8 | **High** | slowness | Default limit 100 000, no `X-Limit` cap, no LIMIT with `NoCount`/`skipcount`, unbounded `io.ReadAll` of `[post_body]`, 15-minute timeout, full `COUNT(1)` per list request |
|
||||||
|
| 9 | **Medium** | locking | `HookRegistry` map and `variablesCallback` are unsynchronized; `Register`/`Clear*` race with `Execute` |
|
||||||
|
| 10 | **Medium** | security | Dollar-quote substitution (`[post_body]`, `[user]`, `[method]`, …) skips backslash escaping; `[id_session]` substituted unquoted |
|
||||||
|
| 11 | **Low** | correctness | `Content-Range` offset comes from the `offset` query param only; header offset ignored |
|
||||||
|
| 12 | **Low** | correctness | `X-Select-Fields`/`X-Not-Select-Fields` accepted but no-ops; `sort` `-col` negates instead of DESC |
|
||||||
|
| 13 | **Low** | panic / logging | Recovery is handler-level only; `Serving: Records` logged at Info per request; hook/filter strings logged unscrubbed (X8) |
|
||||||
|
| 14 | **Low** | correctness | `BeforeResponse` runs post-commit on the pool, not the tx (see `audit/single_tran.md`) |
|
||||||
|
| 15 | **Low** | security | Security adapter hard-codes schema `public` and entity `sql_query`; per-entity rules cannot be applied |
|
||||||
|
| 16 | **Info** | testing | Regexes compiled per call (`ValidSQL`, `sqlStripStringLiterals`); no `-race` in CI (X1); no hostile-input tests for findings 1-4 |
|
||||||
|
|
||||||
|
## 1. Raw SQL fragments behind a keyword denylist — High
|
||||||
|
|
||||||
|
`ApplyFilters` (`parameters.go:283-297`) passes `X-Custom-SQL-W` and
|
||||||
|
`X-Custom-SQL-Or` through `ValidSQL(..., "select")` and splices the result into
|
||||||
|
the query. `sort` goes the same way into `ORDER BY` (`function_api.go:~226`).
|
||||||
|
The denylist (`function_api.go:964-979`) removes `;`, `--`, `/*`, `*/`, `xp_`,
|
||||||
|
`sp_` and a few keywords **followed by a space**. It is not a parser:
|
||||||
|
- Subqueries, function calls (`pg_sleep`, `pg_read_*` where permitted),
|
||||||
|
`SELECT` itself and `)` are not blocked; a `)` can close the
|
||||||
|
`COUNT(1) FROM (%s) cnts` wrapper (`function_api.go:~241`).
|
||||||
|
- Keywords are removed rather than rejected, so input can be shaped so that
|
||||||
|
removal assembles a different token.
|
||||||
|
- Whitespace variants (tab, newline) bypass the `keyword␠` patterns.
|
||||||
|
|
||||||
|
Whether the raw fragments are reachable is decided by the handler; they are
|
||||||
|
parsed whenever `ParseParameters` runs, i.e. always. Fix: drop the two headers
|
||||||
|
from the wire contract, or accept only a column/operator/value structure built
|
||||||
|
by the server; validate `sort` against `^[A-Za-z0-9_.]+( (ASC|DESC))?(,…)*$`
|
||||||
|
and ideally an allowlist of columns.
|
||||||
|
|
||||||
|
## 2. OR precedence widens results — High
|
||||||
|
|
||||||
|
`sqlQryWhereOr` (`parameters.go:381-411`) rewrites `WHERE a AND b` into
|
||||||
|
`WHERE a AND b OR (c)`. SQL evaluates `AND` first, so the result is
|
||||||
|
`(a AND b) OR c`: any row satisfying `c` is returned regardless of `a`/`b`.
|
||||||
|
Where `a` is a tenant or ownership predicate in the template, a client-supplied
|
||||||
|
OR condition (`X-SearchOr`, `X-Custom-SQL-Or`, search operator with logic OR)
|
||||||
|
returns other tenants' rows. Verified with `ParseParameters` + `ApplyFilters`
|
||||||
|
on generated headers. Fix: wrap the existing WHERE body in parentheses before
|
||||||
|
appending `OR (...)`, or build a predicate tree.
|
||||||
|
|
||||||
|
## 3. Substring-based clause location — Medium
|
||||||
|
|
||||||
|
Both helpers use `strings.Index` on `" where "`, `" group by"`, `" order by"`,
|
||||||
|
`" limit "` over the whole lower-cased query. A match inside a string literal,
|
||||||
|
a subquery, a CTE or a column alias selects the wrong insertion point, and
|
||||||
|
`wherePos > 0` decides AND-append vs. new WHERE on the first match anywhere.
|
||||||
|
`ApplyDistinct` (`parameters.go:363-378`) similarly inserts after the first
|
||||||
|
`SELECT` substring, and the ORDER BY test (`function_api.go:~224`) compares the
|
||||||
|
first `order by` to the first `from `. `sqlStripStringLiterals` exists
|
||||||
|
(`function_api.go:858`) but is not used by these helpers.
|
||||||
|
|
||||||
|
## 4. Filter handling inconsistencies — Medium
|
||||||
|
|
||||||
|
- `ApplyFilters` builds `col = value` for `X-FieldFilter` without quoting the
|
||||||
|
value (`parameters.go:248-250`), so a string value becomes a column reference
|
||||||
|
(`status = active`). `mergeHeaderParams` quotes the same filter, so the
|
||||||
|
`SqlQuery` path applies it twice with different semantics.
|
||||||
|
- `RequestParameters.SearchOps` is a map keyed by column; two operators on one
|
||||||
|
column overwrite each other and map iteration order makes the generated WHERE
|
||||||
|
non-deterministic.
|
||||||
|
|
||||||
|
## 5. Information disclosure in errors — High
|
||||||
|
|
||||||
|
- `sendError` (`function_api.go:1150-1172`) sets `Detail = err.Error()` and,
|
||||||
|
for `*common.SQLError`, `SQL` = the final statement, including the template,
|
||||||
|
substituted values and any injected fragment. Used by every failure path
|
||||||
|
(`query_failed`, `count_failed`, `hook_error`).
|
||||||
|
- Panic recovery in `SqlQueryList` (`:80-86`) and `SqlQuery` (`:433-439`) calls
|
||||||
|
`http.Error(w, fmt.Sprintf("Internal server error: %v", err), 500)`; the
|
||||||
|
panic value reaches the client. Same class as `middleware` finding 4.
|
||||||
|
Fix: log server-side, return a generic message plus a request id.
|
||||||
|
|
||||||
|
## 6. Auth hook abort returns 400, hooks opt-in — High
|
||||||
|
|
||||||
|
`RegisterSecurityHooks` (`security_adapter.go:14-55`) sets `Abort`,
|
||||||
|
`AbortCode=401` **and returns an error**. `HookRegistry.Execute`
|
||||||
|
(`hooks.go:113-137`) returns the error before it evaluates `Abort`, and the
|
||||||
|
handler maps that to `sendError(400, "hook_error", …)`
|
||||||
|
(`function_api.go:~202`). The 401 branch in the handler is only reachable for
|
||||||
|
hooks that set `Abort` without returning an error. Clients therefore see 400
|
||||||
|
with `Detail: "hook execution failed: authentication required"`.
|
||||||
|
|
||||||
|
Also: without `RegisterSecurityHooks` there is no authentication at all; a
|
||||||
|
missing user context is replaced with `UserID 0, "anonymous"`
|
||||||
|
(`function_api.go:~103`) and the request proceeds. Fix: return nil after
|
||||||
|
setting `Abort` in the auth hooks, or have the handler honour `AbortCode` when
|
||||||
|
the error wraps an abort; consider fail-closed by default.
|
||||||
|
|
||||||
|
## 7. Header/param decoding — Medium
|
||||||
|
|
||||||
|
`decodeValue` (`parameters.go:203`) calls `restheadspec.DecodeParam` and drops
|
||||||
|
the error. `DecodeParam` replaces all `ZIP_`/`__` occurrences and decodes
|
||||||
|
recursively with no depth limit, so one value can force repeated base64/gzip
|
||||||
|
work (decompression amplification, since size is not capped). Header keys are
|
||||||
|
matched with `HasPrefix`, so `X-SearchOp-<anything>` variants and unrelated
|
||||||
|
headers with the same prefix are interpreted.
|
||||||
|
|
||||||
|
## 8. Unbounded resource use — High
|
||||||
|
|
||||||
|
- `parameters.go:54` default `Limit: 100000`; `X-Limit` and `limit` accept any
|
||||||
|
positive integer.
|
||||||
|
- In `SqlQueryList` the `LIMIT`/`OFFSET` clause is added **only inside
|
||||||
|
`if !options.NoCount`** (`function_api.go:~232-251`); `NoCount` or
|
||||||
|
`X-SkipCount` returns the whole result set.
|
||||||
|
- `COUNT(1) FROM (<full query>)` runs on every list request (double execution
|
||||||
|
cost).
|
||||||
|
- `[post_body]` uses `io.ReadAll(r.Body)` (`function_api.go:913`) with no
|
||||||
|
`http.MaxBytesReader`; the body is also embedded into the SQL text.
|
||||||
|
- `context.WithTimeout(…, 15*time.Minute)` (`:91`, `:444`) holds a transaction
|
||||||
|
and pooled connection for up to 15 minutes per request.
|
||||||
|
- `ValidSQL` and `sqlStripStringLiterals` compile regexes on each call.
|
||||||
|
Fix: hard cap on limit, always apply LIMIT, cap body size, configurable timeout.
|
||||||
|
|
||||||
|
## 9. Unsynchronized registry — Medium
|
||||||
|
|
||||||
|
`HookRegistry.hooks` (`hooks.go`) is a plain map; `Register`, `Clear`,
|
||||||
|
`ClearAll` mutate it while `Execute` reads it from request goroutines. Safe
|
||||||
|
only if all registration completes before serving. `Handler.variablesCallback`
|
||||||
|
(`function_api.go:60-68`) has the same property. Fix: `sync.RWMutex` and copy-on-
|
||||||
|
read, or document and enforce "register before serve".
|
||||||
|
|
||||||
|
## 10. Dollar-quote substitution — Medium
|
||||||
|
|
||||||
|
`safeSubstituteVar` returns the raw value when the placeholder is adjacent to
|
||||||
|
`$` (`function_api.go:1044-1049`), so neither backslash nor quote escaping
|
||||||
|
applies. The tag is neutralised only for `$M$`, `$PBODY$` and the equivalents
|
||||||
|
in `replaceMetaVariables`; a caller-supplied value in a template that uses a
|
||||||
|
different tag (or `$$`) is not. `isInsideDollarQuote` inspects only the first
|
||||||
|
occurrence of the placeholder. `[id_session]` is replaced without any quoting
|
||||||
|
(`function_api.go:~900`); its source is the auth layer, but it becomes an
|
||||||
|
injection point if a session token format allows quotes.
|
||||||
|
|
||||||
|
## 11-12. Behavioural defects — Low
|
||||||
|
|
||||||
|
- `Content-Range` offset uses only `r.URL.Query().Get("offset")`
|
||||||
|
(`function_api.go:~319`) while the applied offset can come from
|
||||||
|
`X-Offset`; the reported range is wrong for header-driven paging.
|
||||||
|
- `ApplyFieldSelection` (`parameters.go:226-241`) only logs; the headers have
|
||||||
|
no effect. `sort=-col` is not converted to DESC; it is emitted as `ORDER BY
|
||||||
|
-col`, which negates the column value.
|
||||||
|
|
||||||
|
## 13. Panic handling and logging — Low
|
||||||
|
|
||||||
|
Recovery exists per handler only (no middleware-level recovery for hooks run
|
||||||
|
outside), and the stack is logged via `logger.Error`, which forwards to Sentry
|
||||||
|
unscrubbed (X8). `logger.Info("Serving: Records …")` runs on every list request.
|
||||||
|
`logger.Debug` lines include the generated filter SQL and attacker-supplied
|
||||||
|
values. Hook failures log `err` with attacker-influenced text.
|
||||||
|
|
||||||
|
## 14. `BeforeResponse` outside the transaction — Low
|
||||||
|
|
||||||
|
`BeforeResponse` executes after `RunInTransaction` returns, with
|
||||||
|
`hookCtx.Tx = h.db` (`function_api.go:~336-343`, `:~640`). A hook that writes
|
||||||
|
cannot be rolled back with the query, and a failure returns 500 after the work
|
||||||
|
committed. Tracked in `audit/single_tran.md`.
|
||||||
|
|
||||||
|
## 15. Security adapter — Low
|
||||||
|
|
||||||
|
`funcSpecSecurityContext.GetSchema()` returns `"public"` and `GetEntity()`
|
||||||
|
returns `"sql_query"` for every endpoint (`security_adapter.go:84-92`), so
|
||||||
|
column/row security rules keyed by entity cannot distinguish funcspec endpoints.
|
||||||
|
`GetModel`, `GetQuery`, `SetQuery` are stubs.
|
||||||
|
|
||||||
|
## 16. Testing — Info
|
||||||
|
|
||||||
|
Tests cover handler flow, hooks and parameter parsing. No test exercises the
|
||||||
|
hostile inputs of findings 1-4 or the 401-vs-400 outcome. There is no `-race`
|
||||||
|
job in CI (X1). The earlier note about a failing
|
||||||
|
`TestReplaceMetaVariables/Replace_[user]` no longer reproduces: the package
|
||||||
|
passes today.
|
||||||
|
|
||||||
|
## Cross-references
|
||||||
|
|
||||||
|
X1 (no `-race`), X7 (inconsistent panic handling), X8 (logger forwards to
|
||||||
|
Sentry unscrubbed), `middleware` finding 4 (panic value in body),
|
||||||
|
`audit/single_tran.md` (post-commit hooks).
|
||||||
@@ -0,0 +1,285 @@
|
|||||||
|
# Audit — `pkg/logger`
|
||||||
|
|
||||||
|
- **Date:** 2026-09-29
|
||||||
|
- **Scope:** `pkg/logger/logger.go` (211 LOC, 1 file, no tests)
|
||||||
|
- **Axes:** thread locking/waiting · slowness · security · panic handling & logging
|
||||||
|
- **Threat model:** hostile internet client; request bodies, headers, params and identifiers are attacker-controlled.
|
||||||
|
|
||||||
|
## Summary
|
||||||
|
|
||||||
|
`pkg/logger` is a thin package-global wrapper over `zap.SugaredLogger` plus a fan-out to
|
||||||
|
`pkg/errortracking`. It is the single most widely imported package in the repo, so its defects
|
||||||
|
are systemic. Two classes of problem dominate: **unsynchronised global mutable state** (a real
|
||||||
|
data race between logger re-initialisation and request-path logging), and **unbounded,
|
||||||
|
unsampled, unscrubbed egress of formatted messages to a third-party error tracker** on every
|
||||||
|
`Warn`/`Error` call — which under hostile input is both a data-leak and a cost/latency
|
||||||
|
amplification channel.
|
||||||
|
|
||||||
|
There are **zero tests** in this package.
|
||||||
|
|
||||||
|
| # | Severity | Axis | Finding |
|
||||||
|
|---|----------|------|---------|
|
||||||
|
| 1 | **High** | Locking | Unsynchronised writes to `Logger` / `errorTracker` globals race with every log call |
|
||||||
|
| 2 | **High** | Security | Every `Warn`/`Error` message is shipped verbatim to Sentry — no scrubbing, no allowlist |
|
||||||
|
| 3 | **High** | Slowness | No rate limit, sampling or dedup on error-tracker fan-out; attacker-triggerable |
|
||||||
|
| 4 | **High** | Panic | `CatchPanic` swallows panics unconditionally — and both call sites are security enforcement functions (fail-open) |
|
||||||
|
| 5 | Medium | Slowness | `debug.Stack()` + full stack stringification on every recovered panic |
|
||||||
|
| 6 | Medium | Security | `log.Printf(template, args...)` fallback is a format-string sink for caller-supplied text |
|
||||||
|
| 7 | Medium | Security | No CRLF/control-char sanitisation on the stdlib fallback path → log injection |
|
||||||
|
| 8 | Medium | Correctness | `Info`/`Debug` do not strip `context.Context` args; `Warn`/`Error` do |
|
||||||
|
| 9 | Low | Correctness | `UpdateLogger` leaks the previous zap logger / file descriptor |
|
||||||
|
| 10 | Low | Correctness | No `Sync()` exported → buffered log lines lost on exit |
|
||||||
|
| 11 | Low | Slowness | `os.Getpid()` called on every log line |
|
||||||
|
| 12 | Low | Observability | `UpdateLogger` build failure degrades silently to stdlib `log` |
|
||||||
|
|
||||||
|
## Resolution status (2026-09-30)
|
||||||
|
|
||||||
|
- **#1** — Fixed (earlier race work): `stateMu` RWMutex with `getLogger`/`swapLogger`/`getErrorTracker`; the exported `Logger` var is kept for compatibility
|
||||||
|
- **#2** — Fixed: messages are scrubbed before `CaptureMessage` (URL credentials, `password=`/`token=`/`secret=`/`api_key=` values, `Bearer`/`Basic` tokens). Local logs are unchanged. Sentry `BeforeSend` and structured-field allowlisting are not done
|
||||||
|
- **#3** — Partly fixed: global token bucket (burst 50, 20/s) plus per-severity/template dedup (1s, 1024 keys). Panics are not limited. The `error_tracking.sample_rate` default (Sentry maps 0 to 1.0) is still unset in `config/manager.go`
|
||||||
|
- **#4** — Partly fixed: `CatchPanicRethrow` added. `pkg/security/provider.go:302` and `:443` still use the swallowing `CatchPanic`; left for the security audit pass
|
||||||
|
- **#5** — Fixed: stack captured with `runtime.Stack` into a 16 KiB buffer. Per-fingerprint panic rate limiting not done
|
||||||
|
- **#6** — Fixed: `Info`/`Debug` format first and fall back with `log.Printf("%s", ...)`. `gosec` was enabled separately
|
||||||
|
- **#7** — Fixed: CR/LF and other control characters are escaped on the stdlib fallback path
|
||||||
|
- **#8** — Fixed: `Info`/`Debug` strip `context.Context` args
|
||||||
|
- **#9** — Fixed: the replaced logger is synced on `UpdateLogger`
|
||||||
|
- **#10** — Fixed: `logger.Sync()` added. Not yet called from the server shutdown path
|
||||||
|
- **#11** — Fixed: PID cached in a package var
|
||||||
|
- **#12** — Partly fixed: `UpdateLoggerE` returns the build error and a failed build keeps the previous logger. `Init` still returns nothing
|
||||||
|
- Tests: `pkg/logger/logger_test.go` (run with `-race`).
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Findings
|
||||||
|
|
||||||
|
### 1. Unsynchronised global mutable state — data race (High, Locking)
|
||||||
|
|
||||||
|
`logger.go:14-15`
|
||||||
|
|
||||||
|
```go
|
||||||
|
var Logger *zap.SugaredLogger
|
||||||
|
var errorTracker errortracking.Provider
|
||||||
|
```
|
||||||
|
|
||||||
|
`Logger` is written by `Init` → `UpdateLogger` (`logger.go:51`) and by `UpdateLoggerPath`
|
||||||
|
(`logger.go:29`). `errorTracker` is written by `InitErrorTracking` (`logger.go:57`) and read by
|
||||||
|
`GetErrorTracker`, `CloseErrorTracking`, `Warn`, `Error`, `CatchPanicCallback`, `HandlePanic`.
|
||||||
|
|
||||||
|
Every read site (`logger.go:100`, `108`, `123`, `139`, `156`, `199`) is unguarded. There is no
|
||||||
|
mutex, no `atomic.Value`, no `sync.Once`.
|
||||||
|
|
||||||
|
- **Benign case:** everything is initialised once in `main` before goroutines start. Then it's fine.
|
||||||
|
- **Real case:** `UpdateLoggerPath` is an exported, runtime-callable API. A config reload, a
|
||||||
|
log-rotation hook, or a test helper calling it while HTTP handlers log concurrently is an
|
||||||
|
unsynchronised write to an interface value and a pointer, concurrent with reads. Under the Go
|
||||||
|
memory model this is undefined behaviour; in practice a torn interface read (type word from the
|
||||||
|
new value, data word from the old) faults.
|
||||||
|
- `CloseErrorTracking` (`logger.go:69`) does a read-check-then-use on `errorTracker` with no
|
||||||
|
guard, so a concurrent `InitErrorTracking(nil)` yields a nil-interface dereference inside
|
||||||
|
`Flush`.
|
||||||
|
|
||||||
|
**Recommendation:** store both behind `atomic.Pointer`/`atomic.Value` (or an `sync.RWMutex`),
|
||||||
|
and gate first-time init behind `sync.Once`. Run the test suite with `-race` — see finding 12 of
|
||||||
|
`audit/pkg/config.audit.md` for the same pattern in the config singleton.
|
||||||
|
|
||||||
|
### 2. Unscrubbed message egress to third-party error tracker (High, Security)
|
||||||
|
|
||||||
|
`logger.go:110-118` and `logger.go:126-134`
|
||||||
|
|
||||||
|
```go
|
||||||
|
message := fmt.Sprintf(template, remainingArgs...)
|
||||||
|
...
|
||||||
|
errorTracker.CaptureMessage(ctx, message, errortracking.SeverityError, ...)
|
||||||
|
```
|
||||||
|
|
||||||
|
*Every* `Warn` and `Error` call in the entire codebase has its fully-formatted message sent to
|
||||||
|
the configured provider (Sentry, in practice). There is no allowlist, no redaction hook, and
|
||||||
|
`pkg/errortracking/sentry.go` configures no `BeforeSend` scrubber.
|
||||||
|
|
||||||
|
Concretely, formatted error strings across `pkg/` embed: SQL fragments and bound values, DB
|
||||||
|
connection strings, schema/table/column identifiers, filter expressions built from request
|
||||||
|
input, and raw request bodies in a few handlers. Under the hostile-client threat model this is
|
||||||
|
two problems at once:
|
||||||
|
|
||||||
|
- **Outbound data leak:** secrets that appear in wrapped driver errors (DSNs, credentials from
|
||||||
|
`pq`/`pgx` connect failures) leave the trust boundary to a SaaS endpoint.
|
||||||
|
- **Attacker-controlled exfil channel:** an attacker who can shape a value that ends up in an
|
||||||
|
error message gets that value written to a third-party system — useful for exfiltrating data
|
||||||
|
read out of the DB via an induced error.
|
||||||
|
|
||||||
|
**Recommendation:** add a redaction step before `CaptureMessage`/`CapturePanic` (regex-strip
|
||||||
|
DSN/`password=`/bearer-token shapes at minimum), and set Sentry's `BeforeSend` as a second
|
||||||
|
layer. Prefer passing structured fields with an explicit allowlist over shipping the rendered
|
||||||
|
string.
|
||||||
|
|
||||||
|
### 3. No rate limiting or sampling on error-tracker fan-out (High, Slowness)
|
||||||
|
|
||||||
|
`logger.go:113`, `logger.go:129`
|
||||||
|
|
||||||
|
An unauthenticated request that reliably produces one `Error` log (a malformed filter, an unknown
|
||||||
|
column, a bad JSON body — all of which the spec handlers log at error level) becomes one Sentry
|
||||||
|
event. At even modest request rates this means:
|
||||||
|
|
||||||
|
- Sentry quota burn → a direct billing-DoS.
|
||||||
|
- `sentry-go` enqueues onto a bounded worker queue; once saturated events are dropped, so the
|
||||||
|
*real* errors are the ones lost.
|
||||||
|
- `pkg/errortracking/sentry.go:34` passes `SampleRate` straight through from config, and
|
||||||
|
`config/manager.go` sets **no default** for it. `sentry-go@v0.46.2` `client.go:339` maps
|
||||||
|
`SampleRate == 0.0` → `1.0`, so the out-of-the-box behaviour is *send 100% of events*.
|
||||||
|
|
||||||
|
**Recommendation:** default `error_tracking.sample_rate` to something < 1.0 for the message path,
|
||||||
|
and put a token-bucket or a fingerprint-dedup in front of `CaptureMessage`. Keep panics at 100%.
|
||||||
|
|
||||||
|
### 4. `CatchPanic` swallows panics unconditionally, fail-open at both call sites (High, Panic handling)
|
||||||
|
|
||||||
|
`logger.go:145-176`
|
||||||
|
|
||||||
|
```go
|
||||||
|
func CatchPanicCallback(location string, cb func(err any), args ...interface{}) func() {
|
||||||
|
...
|
||||||
|
if err := recover(); err != nil { ... if cb != nil { cb(err) } }
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
The recovered value is logged and then discarded. There is no variant that logs-and-re-panics
|
||||||
|
and no way for the caller to signal "this panic means state is corrupt, take the process down".
|
||||||
|
|
||||||
|
This is the right default for an HTTP handler boundary. The two current call sites are **not**
|
||||||
|
handler boundaries:
|
||||||
|
|
||||||
|
- `pkg/security/provider.go:302` — `defer logger.CatchPanic("ApplyColumnSecurity")()`
|
||||||
|
- `pkg/security/provider.go:443` — `defer logger.CatchPanic("GetRowSecurityTemplate")()`
|
||||||
|
|
||||||
|
Both are *security enforcement* functions. Swallowing a panic there means the column-security
|
||||||
|
filter or row-security template silently does not get applied, and the caller — which has no way
|
||||||
|
to learn a panic occurred, since `CatchPanic` returns nothing and sets no error — proceeds as if
|
||||||
|
security was applied. That is a fail-open security control; see
|
||||||
|
`audit/pkg/security.audit.md` for the full write-up of those two sites.
|
||||||
|
|
||||||
|
Separately: a panic while a mutex is held does not release that mutex unless an intervening
|
||||||
|
`defer Unlock` exists, so swallowing converts a crash into a permanent deadlock at any
|
||||||
|
lock-holding call site.
|
||||||
|
|
||||||
|
**Recommendation:** add `CatchPanicRethrow(location string)` for internal use and reserve the
|
||||||
|
swallowing form for the outermost request/goroutine boundary. Document which is which.
|
||||||
|
|
||||||
|
### 5. Full stack capture on every recovered panic (Medium, Slowness)
|
||||||
|
|
||||||
|
`logger.go:158` and `logger.go:197`
|
||||||
|
|
||||||
|
```go
|
||||||
|
callstack := debug.Stack()
|
||||||
|
```
|
||||||
|
|
||||||
|
`debug.Stack()` stops the world briefly and allocates; `HandlePanic` then formats the whole trace
|
||||||
|
into a string *and* ships it to Sentry. Because panics on the request path are recovered rather
|
||||||
|
than fatal (finding 4), an attacker who finds one reliably-panicking input turns each request
|
||||||
|
into a stack capture + string build + network event. That is a solid amplification factor over a
|
||||||
|
normal request.
|
||||||
|
|
||||||
|
**Recommendation:** cap the captured stack (`runtime.Stack` into a fixed 8–16 KiB buffer rather
|
||||||
|
than `debug.Stack()`'s grow-until-it-fits loop), and rate-limit identical panic fingerprints.
|
||||||
|
|
||||||
|
### 6. Format-string sink in the stdlib fallback (Medium, Security)
|
||||||
|
|
||||||
|
`logger.go:100`, `logger.go:142` (and `108`/`123` with `"%s"`, correctly)
|
||||||
|
|
||||||
|
```go
|
||||||
|
func Info(template string, args ...interface{}) {
|
||||||
|
if Logger == nil {
|
||||||
|
log.Printf(template, args...) // template is the caller's, args may be empty
|
||||||
|
```
|
||||||
|
|
||||||
|
`Info` and `Debug` pass `template` directly to `log.Printf`. If any caller ever does
|
||||||
|
`logger.Info(someUserString)` — the idiomatic-looking single-argument call — a `%s` or `%n` in
|
||||||
|
that string is interpreted as a verb, producing `%!s(MISSING)` garbage and mangled logs. Note
|
||||||
|
`Warn`/`Error` already avoid this on the fallback path by using `log.Printf("%s", message)`;
|
||||||
|
`Info`/`Debug` do not.
|
||||||
|
|
||||||
|
A grep of `pkg/` found **no** current single-argument call sites, so this is a latent API footgun
|
||||||
|
rather than a live bug — but it is one that costs one line to close.
|
||||||
|
|
||||||
|
**Recommendation:** mirror `Warn`'s shape: format first, then `log.Printf("%s", message)`.
|
||||||
|
`govet` runs by default under golangci-lint v2's standard set, and its `printf` analyser infers
|
||||||
|
wrappers like these — so once the fallback is fixed, call sites are checked at build time for
|
||||||
|
free. (Note `gosec` is *not* in `.golangci.json`'s `linters.enable` list; it appears only in the
|
||||||
|
exclusion rules. Worth enabling repo-wide.)
|
||||||
|
|
||||||
|
### 7. No log-injection sanitisation on the fallback path (Medium, Security)
|
||||||
|
|
||||||
|
On the zap path, the JSON encoder escapes newlines and control characters, so injected content
|
||||||
|
can't forge a log record. On the `Logger == nil` fallback path, `log.Printf` writes raw bytes: a
|
||||||
|
value containing `\n2026-09-29 ... level=info authorized=true` forges a plausible second log
|
||||||
|
line. Combined with finding 12 (silent degradation to the fallback path) this is reachable
|
||||||
|
without the operator noticing the encoder changed.
|
||||||
|
|
||||||
|
**Recommendation:** strip/escape `\r`, `\n` and other C0 control characters from formatted
|
||||||
|
messages before the stdlib write.
|
||||||
|
|
||||||
|
### 8. `Info`/`Debug` don't strip `context.Context` arguments (Medium, Correctness)
|
||||||
|
|
||||||
|
`extractContext` (`logger.go:79-98`) exists precisely so callers can pass a `ctx` as a trailing
|
||||||
|
variadic arg. `Warn` (`logger.go:106`) and `Error` (`logger.go:121`) call it. `Info`
|
||||||
|
(`logger.go:99`) and `Debug` (`logger.go:137`) **do not** — they pass every arg to `Sprintf`.
|
||||||
|
|
||||||
|
So `logger.Info("saved %s", name, ctx)` renders as
|
||||||
|
`saved widget%!(EXTRA *context.valueCtx=context.Background...)`, dumping the context's contents
|
||||||
|
(which in this codebase carry auth/tenant values) into the log line. That is both noise and a
|
||||||
|
minor disclosure.
|
||||||
|
|
||||||
|
**Recommendation:** call `extractContext` in all four level functions for uniform behaviour.
|
||||||
|
|
||||||
|
### 9. `UpdateLogger` leaks the previous logger (Low)
|
||||||
|
|
||||||
|
`logger.go:37-53` builds a new zap logger and overwrites `Logger` without calling `Sync()`/close
|
||||||
|
on the old one. `UpdateLoggerPath` opens a new file sink each call; repeated calls leak a file
|
||||||
|
descriptor each time and buffered lines in the old logger are lost.
|
||||||
|
|
||||||
|
### 10. No `Sync()` on shutdown (Low)
|
||||||
|
|
||||||
|
Nothing in the package exposes `Logger.Sync()`, and `CloseErrorTracking` (`logger.go:69`) flushes
|
||||||
|
only the error tracker. zap buffers writes to file sinks, so the last lines before exit — often
|
||||||
|
the interesting ones — are dropped. Add `func Sync() error` and call it from the server's
|
||||||
|
shutdown path alongside `CloseErrorTracking`.
|
||||||
|
|
||||||
|
### 11. `os.Getpid()` per log line (Low, Slowness)
|
||||||
|
|
||||||
|
`logger.go:102`, `111`, `127`, `140`, `165`, `202`. On Linux `getpid` is cached by the runtime so
|
||||||
|
this is cheap, but the PID cannot change for the life of the process — cache it in a package var
|
||||||
|
and drop six calls from the hot path.
|
||||||
|
|
||||||
|
### 12. Silent degradation when the logger fails to build (Low, Observability)
|
||||||
|
|
||||||
|
`logger.go:45-49`
|
||||||
|
|
||||||
|
```go
|
||||||
|
logger, err := config.Build()
|
||||||
|
if err != nil { log.Print(err); return }
|
||||||
|
```
|
||||||
|
|
||||||
|
`Logger` stays `nil`, so the whole process silently falls back to unstructured stdlib logging
|
||||||
|
(and thereby onto the format-string and log-injection paths of findings 6 and 7) with a single
|
||||||
|
line of warning that itself goes to stderr. A bad `logger.path` in config (unwritable directory)
|
||||||
|
triggers exactly this.
|
||||||
|
|
||||||
|
**Recommendation:** return the error from `Init`/`UpdateLogger` and let the caller decide whether
|
||||||
|
to fail startup.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## What looks right
|
||||||
|
|
||||||
|
- `extractContext` correctly ignores second and subsequent contexts rather than fighting over them.
|
||||||
|
- `Warn`/`Error` use `log.Printf("%s", message)` on the fallback path — the safe form.
|
||||||
|
- `HandlePanic` returns an `error` rather than swallowing, which lets callers convert a panic into
|
||||||
|
a normal error return. This is the better of the two panic idioms in the package.
|
||||||
|
- The `errortracking.Provider` indirection means a nil/noop provider is always safe to call.
|
||||||
|
|
||||||
|
## Suggested follow-up
|
||||||
|
|
||||||
|
1. Guard the two globals (finding 1) — prerequisite for running the suite under `-race`.
|
||||||
|
2. Add redaction + sampling in front of the error-tracker fan-out (findings 2, 3).
|
||||||
|
3. Split `CatchPanic` into swallow/rethrow variants and re-audit the ~60 `recover()` sites
|
||||||
|
listed in the other package audits against the split (finding 4).
|
||||||
|
4. Add a test file. Minimum: concurrent `UpdateLogger` + `Error` under `-race`, `Info` with a
|
||||||
|
`%`-bearing message, and nil-provider paths.
|
||||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,502 @@
|
|||||||
|
# Audit — `pkg/modelregistry`
|
||||||
|
|
||||||
|
- **Date:** 2026-09-29
|
||||||
|
- **Scope:** `pkg/modelregistry/model_registry.go` (381 LOC, 1 file, **no tests**)
|
||||||
|
- **Axes:** thread locking/waiting · slowness · security · panic handling & logging
|
||||||
|
- **Threat model:** hostile internet client. This package holds the `ModelRules` that
|
||||||
|
`pkg/security/hooks.go` consults to authorise read/update/create/delete, so it is **on the
|
||||||
|
authorisation path**.
|
||||||
|
|
||||||
|
## Summary
|
||||||
|
|
||||||
|
This package is the highest-risk find in the audit. It has been deliberately reworked to "never
|
||||||
|
hang" by replacing blocking `Lock`/`RLock` with **bounded `TryLock` retry loops that give up and
|
||||||
|
return a wrong answer** — and because those wrong answers are consumed by
|
||||||
|
`pkg/security/hooks.go` as authorisation decisions, the result is an **authorisation control that
|
||||||
|
fails open under lock contention**.
|
||||||
|
|
||||||
|
The comments in the file are explicit about the trade-off ("falls back to the last known value
|
||||||
|
without synchronization", "the call is a no-op") — so the hazard was known at the time of writing.
|
||||||
|
What appears not to have been traced is where those degraded results end up. They end up in
|
||||||
|
`checkModelUpdateAllowed` / `checkModelDeleteAllowed`, which treat any error as *permit*.
|
||||||
|
|
||||||
|
There are **no tests** in this package and no `-race` coverage of it anywhere.
|
||||||
|
|
||||||
|
| # | Severity | Axis | Finding |
|
||||||
|
|---|----------|------|---------|
|
||||||
|
| 1 | **Critical** | Security + Locking | `GetModel`'s "registry locked" error is consumed by `pkg/security/hooks.go` as *allow by default* → authorisation fails open under write-lock contention |
|
||||||
|
| 2 | **High** | Locking | `GetDefaultRegistry` documents and performs an unsynchronised read of `defaultRegistry` on lock-acquire failure — a data race by design |
|
||||||
|
| 3 | **High** | Locking | `SetDefaultRegistry` silently no-ops after ~20 ms of contention; caller gets no error |
|
||||||
|
| 4 | **High** | Security | `RegisterModelWithRules` is non-atomic: the model is visible with permissive `DefaultModelRules` before its real rules are applied (TOCTOU) |
|
||||||
|
| 5 | Medium | Correctness | `GetAllModels` returns an empty map, and `GetModels` silently skips whole registries, on lock-acquire failure |
|
||||||
|
| 6 | Medium | Locking | `IterateModels` invokes the caller's callback while holding `RLock` → guaranteed self-deadlock if the callback touches the registry |
|
||||||
|
| 7 | Medium | Slowness | `time.Sleep(1ms)` spin loops add up to 20 ms of latency per call and defeat mutex fairness/hand-off |
|
||||||
|
| 8 | Medium | Locking | `defaultRegistry` is read unsynchronised by six package-level functions while `SetDefaultRegistry` writes it under lock |
|
||||||
|
| 9 | Medium | Locking | Inconsistent discipline: `SetModelRules`/`GetModelRules`/`AddRegistry`/`IterateModels` use blocking locks; the rest use try-locks |
|
||||||
|
| 10 | Low | Slowness/Locking | Reflection (`TypeOf`, unwrap loop, `reflect.New`) runs while holding the registry **write** lock |
|
||||||
|
| 11 | Low | Availability | Unbounded unwrap loop: a recursive pointer type (`type T *T`) spins forever holding the write lock (**verified**) |
|
||||||
|
| 12 | Low | Panic | Package has no `recover` anywhere, and calls a caller-supplied callback under a lock (see 6) |
|
||||||
|
| 13 | Low | Security | `DefaultModelRules()` grants `CanRead/Update/Create/Delete: true` — registration without explicit rules is fully mutable |
|
||||||
|
|
||||||
|
## Resolution (2026-09-30)
|
||||||
|
|
||||||
|
Fixed in `pkg/modelregistry/model_registry.go`, `pkg/security/hooks.go`, and new
|
||||||
|
`pkg/modelregistry/model_registry_test.go` (passes under `-race`).
|
||||||
|
|
||||||
|
| # | Status | What changed |
|
||||||
|
|---|--------|--------------|
|
||||||
|
| 1 | **Fixed** | Added sentinels `ErrModelNotFound`, `ErrModelExists`, `ErrInvalidModel` (wrapped, `errors.Is`-friendly). `checkModelUpdateAllowed`/`checkModelDeleteAllowed` now allow-by-default **only** on `ErrModelNotFound`; any other error denies. Lookups can no longer return a "locked" error at all. |
|
||||||
|
| 2 | **Fixed** | `GetDefaultRegistry` uses a plain `RLock`; no unsynchronised fallback. |
|
||||||
|
| 3 | **Fixed** | `SetDefaultRegistry` uses a blocking `Lock` (cannot silently no-op); a nil registry is ignored. |
|
||||||
|
| 4 | **Fixed** | `RegisterModelWithRules` and `RegisterModel` share `registerLocked`, which writes model + rules under one lock acquisition. |
|
||||||
|
| 5 | **Fixed** | `GetAllModels`/`GetModels` use blocking locks and can no longer return empty/partial results due to contention. Signatures unchanged (`GetAllModels` is used through interfaces by resolvespec/restheadspec/openapi). |
|
||||||
|
| 6 | **Fixed** | `IterateModels` iterates a snapshot; the callback runs with no lock held (regression test re-enters the registry). |
|
||||||
|
| 7 | **Fixed** | Try-lock/sleep helpers and `lockRetry*` constants removed. |
|
||||||
|
| 8 | **Fixed** | All package-level functions go through `GetDefaultRegistry()` / `registriesSnapshot()`; `defaultRegistry` is only touched under `registriesMutex`. |
|
||||||
|
| 9 | **Fixed** | One discipline: blocking locks, snapshot-and-release, documented lock order (`registriesMutex` before a registry's mutex). |
|
||||||
|
| 10 | **Fixed** | Reflection/validation (`validateModel`) runs before the write lock is taken. |
|
||||||
|
| 11 | **Fixed** | Unwrap loop capped at 16 levels; `type T *T` now returns `ErrInvalidModel` (tested). |
|
||||||
|
| 12 | **Fixed** | Sentinel errors added. `IterateModels` recovers a callback panic per model, logs it via `logger.HandlePanic` with the model name, and continues; `validateModel` recovers reflection panics and returns `ErrInvalidModel` so registration fails closed. No lock is held during either, so the registry cannot be wedged. |
|
||||||
|
| 13 | **Accepted (decision)** | Allow-by-default retained deliberately: `DefaultModelRules()` still grants read/update/create/delete. Callers wanting restrictions must use `RegisterModelWithRules`/`SetModelRules`. |
|
||||||
|
|
||||||
|
Tests added: sentinel errors, recursive pointer type, pointer normalisation, atomic
|
||||||
|
`RegisterModelWithRules` (concurrent reader never sees permissive rules), re-entrant `IterateModels`,
|
||||||
|
cross-registry `GetModelRulesByName`, and a concurrent `-race` stress test.
|
||||||
|
|
||||||
|
Not changed: the `pkg/security` middleware-wiring question (context fast-path) remains tracked in
|
||||||
|
`audit/pkg/security.audit.md`.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Findings
|
||||||
|
|
||||||
|
### 1. Authorisation fails open under lock contention (Critical, Security + Locking)
|
||||||
|
|
||||||
|
The mechanism spans two packages.
|
||||||
|
|
||||||
|
**Here**, `GetModel` conflates "not found" with "could not lock" into a single `error` return
|
||||||
|
(`model_registry.go:198-210`):
|
||||||
|
|
||||||
|
```go
|
||||||
|
func (r *DefaultModelRegistry) GetModel(name string) (interface{}, error) {
|
||||||
|
if !r.tryRLock() {
|
||||||
|
return nil, fmt.Errorf("failed to get model %s: registry locked", name)
|
||||||
|
}
|
||||||
|
defer r.mutex.RUnlock()
|
||||||
|
|
||||||
|
model, exists := r.models[name]
|
||||||
|
if !exists {
|
||||||
|
return nil, fmt.Errorf("model %s not found", name)
|
||||||
|
}
|
||||||
|
return model, nil
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
`GetModelRulesByName` (`model_registry.go:364-376`) uses `GetModel` as its existence probe:
|
||||||
|
|
||||||
|
```go
|
||||||
|
for _, registry := range registries {
|
||||||
|
if _, err := registry.GetModel(name); err == nil {
|
||||||
|
return registry.GetModelRules(name)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return ModelRules{}, fmt.Errorf("model %s not found in any registry", name)
|
||||||
|
```
|
||||||
|
|
||||||
|
So a `tryRLock` failure makes the registry look like it does not contain the model.
|
||||||
|
|
||||||
|
**In `pkg/security/hooks.go`**, that outcome is interpreted as *permit*
|
||||||
|
(`pkg/security/hooks.go:274-294`, and identically at `:298-318`):
|
||||||
|
|
||||||
|
```go
|
||||||
|
func checkModelUpdateAllowed(secCtx SecurityContext) error {
|
||||||
|
rules, ok := GetModelRulesFromContext(secCtx.GetContext())
|
||||||
|
if !ok {
|
||||||
|
schema := secCtx.GetSchema()
|
||||||
|
entity := secCtx.GetEntity()
|
||||||
|
var err error
|
||||||
|
if schema != "" {
|
||||||
|
rules, err = modelregistry.GetModelRulesByName(fmt.Sprintf("%s.%s", schema, entity))
|
||||||
|
}
|
||||||
|
if err != nil || schema == "" {
|
||||||
|
rules, err = modelregistry.GetModelRulesByName(entity)
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return nil // model not registered, allow by default
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !rules.CanUpdate {
|
||||||
|
return fmt.Errorf("update not allowed for %s", secCtx.GetEntity())
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
Note the context fast-path at `hooks.go:275`: if `NewModelAuthMiddleware` already put rules in the
|
||||||
|
context, the registry is not consulted and this bug does not fire. The registry fallback runs
|
||||||
|
whenever that middleware is absent or did not resolve rules — so the blast radius depends on
|
||||||
|
deployment wiring. `audit/pkg/security.audit.md` covers whether that middleware is mandatory.
|
||||||
|
|
||||||
|
`return nil` from `checkModelUpdateAllowed` means **the update is authorised**. Same for
|
||||||
|
`checkModelDeleteAllowed`. `GetModelRules(name)` for the "found" path also uses a blocking
|
||||||
|
`RLock` (`model_registry.go:253`) — so the two calls in `GetModelRulesByName` don't even use the
|
||||||
|
same locking discipline.
|
||||||
|
|
||||||
|
**Failure scenario.** A model `public.employees` is registered with `CanDelete: false`. A
|
||||||
|
concurrent `RegisterModel` (or `SetModelRules`, or `RegisterModelWithRules`) holds the write lock
|
||||||
|
for longer than `lockRetryAttempts * lockRetryDelay` = 20 ms — which is entirely achievable given
|
||||||
|
finding 10 (reflection under the write lock) and finding 7 (each waiter sleeps in 1 ms
|
||||||
|
increments, so N waiters serialise). During that window every `DELETE` request against
|
||||||
|
`public.employees` has `GetModelRulesByName` return an error, `checkModelDeleteAllowed` return
|
||||||
|
`nil`, and the delete proceeds. The model's `CanDelete: false` is not enforced.
|
||||||
|
|
||||||
|
This is remotely triggerable if any request path can cause a model registration or a rules
|
||||||
|
update; even without that, it is a straightforward race that will fire under load.
|
||||||
|
|
||||||
|
**Recommendation, in order of value:**
|
||||||
|
|
||||||
|
1. Make the security layer **fail closed**: distinguish a sentinel `ErrModelNotFound` from any
|
||||||
|
other error, and only allow-by-default on `ErrModelNotFound`. Any other error must deny.
|
||||||
|
2. Delete the try-lock scheme here entirely and use plain `RLock`/`Lock` (see finding 2 for why
|
||||||
|
the scheme does not achieve its stated goal anyway).
|
||||||
|
3. Separate the existence probe from the rules fetch so `GetModelRulesByName` takes each registry's
|
||||||
|
lock once and returns a typed "found / not found / unavailable" result.
|
||||||
|
|
||||||
|
### 2. `GetDefaultRegistry` races by design (High, Locking)
|
||||||
|
|
||||||
|
`model_registry.go:71-84`
|
||||||
|
|
||||||
|
```go
|
||||||
|
// GetDefaultRegistry returns the current default registry. It uses a
|
||||||
|
// bounded TryRLock instead of a blocking RLock so it can never hang;
|
||||||
|
// if the lock can't be acquired in time it falls back to the last known
|
||||||
|
// value without synchronization.
|
||||||
|
func GetDefaultRegistry() *DefaultModelRegistry {
|
||||||
|
for i := 0; i < lockRetryAttempts; i++ {
|
||||||
|
if registriesMutex.TryRLock() {
|
||||||
|
defer registriesMutex.RUnlock()
|
||||||
|
return defaultRegistry
|
||||||
|
}
|
||||||
|
time.Sleep(lockRetryDelay)
|
||||||
|
}
|
||||||
|
return defaultRegistry
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
The `return defaultRegistry` on line 83 reads a pointer that `SetDefaultRegistry`
|
||||||
|
(`model_registry.go:89-116`) writes under the write lock. The only time this path is taken is
|
||||||
|
precisely when a writer holds or is contending for the lock — i.e. the fallback executes
|
||||||
|
*exactly* in the window where the race is live. The trade is not "hang vs. slightly stale value";
|
||||||
|
it is "block for 20 ms vs. data race", and a torn/`nil` pointer read here means a nil-pointer
|
||||||
|
dereference in the caller.
|
||||||
|
|
||||||
|
The premise is also wrong: a `sync.RWMutex.RLock` that is only ever held for a map lookup cannot
|
||||||
|
"hang". The hang this was written to avoid must have had a different root cause — most likely
|
||||||
|
finding 6 (self-deadlock through `IterateModels`) or a lock-ordering inversion — and the try-lock
|
||||||
|
scheme papers over it rather than fixing it.
|
||||||
|
|
||||||
|
**Recommendation:** revert to `RLock`/`RUnlock`. If a real hang was observed, reproduce it under
|
||||||
|
`-race` and `GODEBUG=gctrace`/`SIGQUIT` stack dump; the fix belongs at the deadlock, not here.
|
||||||
|
|
||||||
|
### 3. `SetDefaultRegistry` silently no-ops (High, Locking)
|
||||||
|
|
||||||
|
`model_registry.go:90-100`
|
||||||
|
|
||||||
|
```go
|
||||||
|
acquired := false
|
||||||
|
for i := 0; i < lockRetryAttempts; i++ {
|
||||||
|
if registriesMutex.TryLock() { acquired = true; break }
|
||||||
|
time.Sleep(lockRetryDelay)
|
||||||
|
}
|
||||||
|
if !acquired {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
The function returns no error. A caller that swaps in a registry — plausibly one with *restrictive*
|
||||||
|
`ModelRules* — has no way to learn the swap did not happen, and continues believing the new
|
||||||
|
registry is in effect. Every subsequent authorisation check consults the old registry's rules.
|
||||||
|
|
||||||
|
`GetModels` (`model_registry.go:319-329`) has the same shape and returns `nil`.
|
||||||
|
|
||||||
|
**Recommendation:** return `error` from `SetDefaultRegistry`; or (better) use a blocking `Lock`,
|
||||||
|
since this is a startup-time operation where blocking is correct.
|
||||||
|
|
||||||
|
### 4. `RegisterModelWithRules` is non-atomic (High, Security)
|
||||||
|
|
||||||
|
`model_registry.go:270-282`
|
||||||
|
|
||||||
|
```go
|
||||||
|
func (r *DefaultModelRegistry) RegisterModelWithRules(name string, model interface{}, rules ModelRules) error {
|
||||||
|
// First register the model
|
||||||
|
if err := r.RegisterModel(name, model); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
// Then set the rules (we need to lock again for rules)
|
||||||
|
r.mutex.Lock()
|
||||||
|
defer r.mutex.Unlock()
|
||||||
|
r.rules[name] = rules
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
`RegisterModel` releases the write lock before returning, and it initialises the model's rules to
|
||||||
|
`DefaultModelRules()` (`model_registry.go:191-194`) — which is **permissive**:
|
||||||
|
`CanRead/CanUpdate/CanCreate/CanDelete` all `true`.
|
||||||
|
|
||||||
|
Between the two lock acquisitions, any concurrent `GetModelRulesByName` sees the model registered
|
||||||
|
with full read/update/create/delete permission, regardless of the restrictive `rules` the caller
|
||||||
|
passed. The comment "we need to lock again for rules" acknowledges the re-lock without noticing
|
||||||
|
the gap it opens.
|
||||||
|
|
||||||
|
**Failure scenario.** `RegisterModelWithRules("public.audit_log", AuditLog{}, ModelRules{CanRead:
|
||||||
|
true})` — intended read-only. A `DELETE /public.audit_log/...` that lands in the window is
|
||||||
|
authorised because `rules.CanDelete` is `true` from the default.
|
||||||
|
|
||||||
|
**Recommendation:** add an unexported `registerLocked(name, model, rules)` that writes both maps
|
||||||
|
under one lock acquisition, and build both public constructors on it. Also change the default
|
||||||
|
initialisation in `RegisterModel` to deny-by-default, or require rules at registration.
|
||||||
|
|
||||||
|
### 5. Degraded results indistinguishable from real results (Medium, Correctness)
|
||||||
|
|
||||||
|
Three functions return a plausible-looking answer when they cannot lock:
|
||||||
|
|
||||||
|
- `GetAllModels` (`model_registry.go:212-215`) — `return make(map[string]interface{})`, i.e. "the
|
||||||
|
registry is empty".
|
||||||
|
- `GetModels` (`model_registry.go:327-329`) — `return nil` on `registriesMutex` failure, and
|
||||||
|
`model_registry.go:336-338` `continue`s past any individual registry it cannot read, returning a
|
||||||
|
**partial** list with no indication of truncation.
|
||||||
|
- `GetDefaultRegistry` — finding 2.
|
||||||
|
|
||||||
|
Consumers of `GetModels`/`GetAllModels` (schema introspection, OpenAPI generation, migration
|
||||||
|
helpers) will emit a document that is missing models, and there is no error to log. Note these two
|
||||||
|
have no callers in `pkg/` today, which is the only reason this is Medium.
|
||||||
|
|
||||||
|
**Recommendation:** return `(T, error)`; never manufacture an empty-but-valid result.
|
||||||
|
|
||||||
|
### 6. `IterateModels` calls a user callback under a read lock (Medium, Locking)
|
||||||
|
|
||||||
|
`model_registry.go:307-314`
|
||||||
|
|
||||||
|
```go
|
||||||
|
func IterateModels(fn func(name string, model interface{})) {
|
||||||
|
defaultRegistry.mutex.RLock()
|
||||||
|
defer defaultRegistry.mutex.RUnlock()
|
||||||
|
|
||||||
|
for name, model := range defaultRegistry.models {
|
||||||
|
fn(name, model)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
`fn` is arbitrary caller code running with `defaultRegistry.mutex` read-held. `sync.RWMutex` is not
|
||||||
|
reentrant, and once a writer is blocked on `Lock` it also blocks *new* readers. So:
|
||||||
|
|
||||||
|
- `fn` calling `modelregistry.RegisterModel` / `SetModelRules` → `Lock` waits for the reader, which
|
||||||
|
is the same goroutine. **Permanent self-deadlock.**
|
||||||
|
- `fn` calling `GetModel` → `tryRLock` fails for 20 ms and returns "registry locked" for every
|
||||||
|
model, which is silent nonsense rather than a deadlock (and feeds finding 1).
|
||||||
|
- `fn` doing anything slow (I/O, a DB call) holds the registry read lock for that whole duration,
|
||||||
|
blocking all registration and — via the blocked-writer rule — all other readers too.
|
||||||
|
|
||||||
|
This is the most likely original cause of the "hang" the try-lock scheme was introduced to work
|
||||||
|
around.
|
||||||
|
|
||||||
|
**Recommendation:** snapshot under the lock, release, then iterate:
|
||||||
|
|
||||||
|
```go
|
||||||
|
func IterateModels(fn func(name string, model interface{})) {
|
||||||
|
reg := GetDefaultRegistry()
|
||||||
|
snapshot := reg.GetAllModels() // takes and releases the lock
|
||||||
|
for name, model := range snapshot {
|
||||||
|
fn(name, model)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
### 7. `time.Sleep` spin loops (Medium, Slowness)
|
||||||
|
|
||||||
|
`tryLock` (`model_registry.go:128-136`), `tryRLock` (`:140-148`), and the inline loops in
|
||||||
|
`GetDefaultRegistry`, `SetDefaultRegistry`, `GetModels`.
|
||||||
|
|
||||||
|
```go
|
||||||
|
for i := 0; i < lockRetryAttempts; i++ {
|
||||||
|
if r.mutex.TryLock() { return true }
|
||||||
|
time.Sleep(lockRetryDelay) // 1ms
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
Problems:
|
||||||
|
|
||||||
|
- **Latency floor.** A contended call costs a multiple of 1 ms even if the lock frees after 10 µs,
|
||||||
|
because the waiter is asleep. A blocking `Lock` would be handed the mutex in microseconds. So the
|
||||||
|
"no-hang" scheme is *slower* in the common contended case, not faster.
|
||||||
|
- **No fairness.** `sync.Mutex` has a starvation-avoidance mode that hands the lock to a waiter
|
||||||
|
queued > 1 ms. `TryLock` participates in none of it, so a try-lock waiter can be starved
|
||||||
|
indefinitely by a stream of blocking `Lock` callers (`SetModelRules`, `AddRegistry`,
|
||||||
|
`IterateModels` all still block) — see finding 9.
|
||||||
|
- **Timer churn.** 20 timer allocations per contended call.
|
||||||
|
- Sleeping in a loop scales badly: 50 concurrent callers each sleep and wake 20 times, producing
|
||||||
|
1000 needless scheduler round-trips for what a mutex does with one park/unpark.
|
||||||
|
|
||||||
|
**Recommendation:** delete the try-lock helpers. If a bounded wait is genuinely required for an
|
||||||
|
SLO, express it as `context`-aware acquisition (a buffered-channel semaphore with a `select` on
|
||||||
|
`ctx.Done()`), which gives a real deadline *and* a real error — not a silent wrong answer.
|
||||||
|
|
||||||
|
### 8. `defaultRegistry` read without the guarding mutex (Medium, Locking)
|
||||||
|
|
||||||
|
`SetDefaultRegistry` writes `defaultRegistry` (`model_registry.go:110`) under `registriesMutex`.
|
||||||
|
These read it **without** taking that mutex:
|
||||||
|
|
||||||
|
- `RegisterModel` (`model_registry.go:288`)
|
||||||
|
- `IterateModels` (`model_registry.go:308`, `311`)
|
||||||
|
- `SetModelRules` (`model_registry.go:354`)
|
||||||
|
- `GetModelRules` (`model_registry.go:359`)
|
||||||
|
- `GetDefaultRegistry`'s fallback (`model_registry.go:83`, finding 2)
|
||||||
|
|
||||||
|
A data race on the pointer, and semantically these functions may operate on the *previous* default
|
||||||
|
registry after a swap — so rules set through `SetModelRules` can land on a registry nobody consults
|
||||||
|
any more.
|
||||||
|
|
||||||
|
**Recommendation:** route every access through one accessor that takes the lock (and make
|
||||||
|
`defaultRegistry` an `atomic.Pointer[DefaultModelRegistry]` if lock-free reads are wanted — that is
|
||||||
|
the correct way to get the "never blocks" property finding 2 was reaching for).
|
||||||
|
|
||||||
|
### 9. Inconsistent locking discipline (Medium, Locking)
|
||||||
|
|
||||||
|
Within one 381-line file:
|
||||||
|
|
||||||
|
| Function | `registriesMutex` | `r.mutex` |
|
||||||
|
|---|---|---|
|
||||||
|
| `GetDefaultRegistry` | `TryRLock` + fallback | — |
|
||||||
|
| `SetDefaultRegistry` | `TryLock`, no-op on fail | — |
|
||||||
|
| `AddRegistry` (`:120`) | blocking `Lock` | — |
|
||||||
|
| `GetModelByName` (`:293`) | blocking `RLock` | via `GetModel` → `TryRLock` |
|
||||||
|
| `GetModelRulesByName` (`:365`) | blocking `RLock` | `TryRLock` then blocking `RLock` |
|
||||||
|
| `GetModels` (`:318`) | `TryRLock`, nil on fail | `tryRLock`, skip on fail |
|
||||||
|
| `RegisterModel` (`:150`) | — | `tryLock`, error on fail |
|
||||||
|
| `GetModel` (`:198`) | — | `tryRLock`, error on fail |
|
||||||
|
| `GetAllModels` (`:212`) | — | `tryRLock`, empty on fail |
|
||||||
|
| `SetModelRules` (`:237`) | — | blocking `Lock` |
|
||||||
|
| `GetModelRules` (`:252`) | — | blocking `RLock` |
|
||||||
|
| `IterateModels` (`:307`) | — | blocking `RLock` |
|
||||||
|
|
||||||
|
Four different failure behaviours for the same class of event. The mix also means the try-lock
|
||||||
|
callers can be starved by the blocking ones (finding 7), so the functions that "can never hang" are
|
||||||
|
the ones most likely to return garbage.
|
||||||
|
|
||||||
|
**Recommendation:** pick one discipline — blocking locks with snapshot-and-release — and apply it
|
||||||
|
uniformly.
|
||||||
|
|
||||||
|
### 10. Reflection under the write lock (Low, Slowness + Locking)
|
||||||
|
|
||||||
|
`RegisterModel` holds `r.mutex` (write) from `model_registry.go:151` through `:195`, and inside
|
||||||
|
that window does `reflect.TypeOf` (`:161`), the unwrap loop (`:169-171`), `reflect.New(...).Elem().Interface()`
|
||||||
|
(`:181`), and another `reflect.TypeOf` (`:185`). None of that touches `r.models`/`r.rules` and none
|
||||||
|
of it needs the lock.
|
||||||
|
|
||||||
|
This directly lengthens the window that makes finding 1 exploitable. Validate first, then take the
|
||||||
|
lock only for the two map writes.
|
||||||
|
|
||||||
|
### 11. Unbounded unwrap loop on a recursive pointer type (Low, Availability)
|
||||||
|
|
||||||
|
`model_registry.go:169-171`
|
||||||
|
|
||||||
|
```go
|
||||||
|
for modelType.Kind() == reflect.Pointer || modelType.Kind() == reflect.Slice || modelType.Kind() == reflect.Array {
|
||||||
|
modelType = modelType.Elem()
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
`type T *T` is legal Go, and `reflect.Type.Elem()` on it returns itself — so the loop never
|
||||||
|
terminates. **Verified experimentally:**
|
||||||
|
|
||||||
|
```go
|
||||||
|
type T *T
|
||||||
|
var x T
|
||||||
|
tt := reflect.TypeOf(x) // main.T
|
||||||
|
for tt.Kind() == reflect.Pointer { tt = tt.Elem() } // spins on main.T forever
|
||||||
|
// → "INFINITE LOOP CONFIRMED after 101 iterations, still main.T"
|
||||||
|
```
|
||||||
|
|
||||||
|
Because the loop runs with the write lock held (finding 10), this doesn't just hang one goroutine —
|
||||||
|
it wedges the registry permanently, at which point every try-lock caller starts returning
|
||||||
|
"registry locked", which via finding 1 means **authorisation fails open for the rest of the process
|
||||||
|
lifetime**.
|
||||||
|
|
||||||
|
Requires a pathological model type, so exploitability is near zero; the fix is a one-line depth cap
|
||||||
|
and it converts a permanent fail-open into an error return.
|
||||||
|
|
||||||
|
**Recommendation:** bound the loop (`for depth := 0; depth < 16 && ...; depth++`) and return an
|
||||||
|
error if the cap is hit.
|
||||||
|
|
||||||
|
### 12. No panic handling at all (Low, Panic handling)
|
||||||
|
|
||||||
|
The package contains **zero** `recover()` calls and never logs — it does not import `pkg/logger`.
|
||||||
|
For a pure data structure that is a defensible choice, with two caveats:
|
||||||
|
|
||||||
|
- `IterateModels` runs a caller callback under a read lock (finding 6). If `fn` panics, the
|
||||||
|
`defer RUnlock` does release the lock, so the registry is not wedged — that part is fine — but
|
||||||
|
the panic propagates to whatever boundary handler exists, and nothing here records which model
|
||||||
|
was being processed. A `logger`-free package can still name the model in a re-panic.
|
||||||
|
- Every failure mode in the package is reported as a `fmt.Errorf` string with no wrapping and no
|
||||||
|
sentinel values, so callers cannot distinguish them (finding 1). That is the panic/error-handling
|
||||||
|
defect that actually matters here.
|
||||||
|
|
||||||
|
**Recommendation:** define `ErrModelNotFound`, `ErrModelExists`, `ErrRegistryUnavailable` as
|
||||||
|
sentinels and wrap them, so `errors.Is` works at the security layer.
|
||||||
|
|
||||||
|
### 13. Permissive default rules (Low, Security)
|
||||||
|
|
||||||
|
`DefaultModelRules()` (`model_registry.go:24-36`) returns `CanRead`, `CanUpdate`, `CanCreate`,
|
||||||
|
`CanDelete` all `true`. `RegisterModel` applies it to any model registered without explicit rules
|
||||||
|
(`model_registry.go:191-194`), and `GetModelRules` falls back to it as well (`model_registry.go:266`).
|
||||||
|
|
||||||
|
The `CanPublic*` flags default to `false` and `SecurityDisabled` to `false`, which is right. But the
|
||||||
|
authenticated-path flags default open, so `RegisterModel(name, m)` — the form used by
|
||||||
|
`pkg/testmodels/business.go` `RegisterTestModels` and the `modelregistry.RegisterModel` convenience wrapper —
|
||||||
|
yields a fully mutable model. Combined with `pkg/security/hooks.go`'s allow-on-error, the system's
|
||||||
|
default posture at every layer is permit.
|
||||||
|
|
||||||
|
**Recommendation:** default to deny and make permissions opt-in, or at minimum log at registration
|
||||||
|
time when a model is registered without explicit rules.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## What looks right
|
||||||
|
|
||||||
|
- The struct-vs-pointer validation in `RegisterModel` (`model_registry.go:160-194`) is careful and
|
||||||
|
well-reasoned: it rejects `nil`, unwraps pointer/slice/array to find the base type, rejects
|
||||||
|
non-struct kinds with a message naming the original type, normalises a pointer/slice input to a
|
||||||
|
zero struct value, and re-checks the final type. The error message even tells the caller to use
|
||||||
|
`MyModel{}` instead of `&MyModel{}`. Good API ergonomics.
|
||||||
|
- Duplicate registration is rejected (`model_registry.go:156-158`) rather than silently overwriting
|
||||||
|
— important, since silent overwrite would be a rules-replacement primitive.
|
||||||
|
- `GetAllModels` returns a **copy** of the map (`model_registry.go:218-222`) rather than the
|
||||||
|
internal one, so callers cannot mutate registry state or race on it after the lock is dropped.
|
||||||
|
This is the pattern the rest of the package should follow.
|
||||||
|
- `GetModelByEntity` (`model_registry.go:225-234`) tries `schema.entity` before bare `entity`,
|
||||||
|
which is the right precedence and matches what `pkg/security/hooks.go` does.
|
||||||
|
- `GetModels` de-duplicates by name across registries (`model_registry.go:335-347`), so
|
||||||
|
registry-order precedence is consistent with `GetModelByName`'s first-match rule.
|
||||||
|
- Every `defer` for an acquired lock is correctly paired; there is no missing-`Unlock` path. The
|
||||||
|
problems here are about *which* lock discipline was chosen, not about leaking locks.
|
||||||
|
|
||||||
|
## Suggested follow-up
|
||||||
|
|
||||||
|
Ordered by risk:
|
||||||
|
|
||||||
|
1. **Make `pkg/security/hooks.go` fail closed** (finding 1). This is the single change that
|
||||||
|
converts a Critical authorisation bypass into a Medium availability issue. It does not require
|
||||||
|
touching this package.
|
||||||
|
2. **Remove the try-lock scheme** (findings 2, 3, 5, 7, 9) and fix the underlying hang by
|
||||||
|
snapshotting in `IterateModels` (finding 6).
|
||||||
|
3. **Make `RegisterModelWithRules` atomic** (finding 4).
|
||||||
|
4. Route `defaultRegistry` access through a single locked accessor or `atomic.Pointer` (finding 8).
|
||||||
|
5. Move reflection out of the write-locked region and cap the unwrap loop (findings 10, 11).
|
||||||
|
6. **Add tests.** This package has none. Priority cases: `-race` test with concurrent
|
||||||
|
`RegisterModel` + `GetModelRulesByName` asserting that rules are *never* observed as permissive
|
||||||
|
for a restrictively-registered model; a test that `GetModelRulesByName` under contention does
|
||||||
|
not return a "not found"-shaped error; `IterateModels` with a callback that calls back into the
|
||||||
|
registry (should not deadlock); sentinel-error assertions.
|
||||||
@@ -0,0 +1,121 @@
|
|||||||
|
# Audit: `pkg/resolvemcp`
|
||||||
|
|
||||||
|
| | |
|
||||||
|
|---|---|
|
||||||
|
| **Package** | `github.com/bitechdev/ResolveSpec/pkg/resolvemcp` |
|
||||||
|
| **Files** | `handler.go` (901), `tools.go` (720), `cursor.go`, `oauth2.go`, `oauth2_server.go`, `annotation.go`, `hooks.go`, `security_hooks.go`, `context.go`, `resolvemcp.go` |
|
||||||
|
| **Tests** | `tools_test.go` (34), `tx_test.go` (207); `go test` passes. No hostile-input tests, no `-race` |
|
||||||
|
| **Audit date** | 2026-09-30 |
|
||||||
|
| **Axes** | thread locking/waiting, slowness, security, panic handling & logging, agent usability |
|
||||||
|
| **Threat model** | hostile or confused MCP client (LLM agent, possibly prompt-injected); tool arguments are attacker-controlled |
|
||||||
|
| **Depth** | targeted (request path, security wiring; verified against source) |
|
||||||
|
|
||||||
|
## Summary
|
||||||
|
|
||||||
|
Every model registers 4 tools + 1 resource (`read_/create_/update_/delete_<schema>_<entity>`), each with an
|
||||||
|
inlined column list, relation list and schema doc. Tool list grows 4N; context cost is
|
||||||
|
paid on every session whether or not the table is used. Replace with fixed meta tools
|
||||||
|
(see Rewrite).
|
||||||
|
|
||||||
|
Security wiring fails open in several places: model rules never reach the hooks,
|
||||||
|
`create` has no rule check, `update` skips `BeforeHandle`, update/delete skip row-level
|
||||||
|
security, and create/update write client-chosen column names. Reads have no size cap.
|
||||||
|
`resolvespec_annotate` is an unauthenticated write channel into agent-visible text.
|
||||||
|
|
||||||
|
## Findings
|
||||||
|
|
||||||
|
| # | Severity | Axis | Finding |
|
||||||
|
|---|---|---|---|
|
||||||
|
| 1 | **High** | security | Model rules set via `RegisterModelWithRules` never reach `security.Check*`: handler uses a private registry that is not `modelregistry.AddRegistry`'d; hooks look up the global list |
|
||||||
|
| 2 | **High** | security | `create` has no rule check: `CheckModelAuthAllowed` only tests `CanPublicCreate`/auth; no `BeforeCreate` hook registered, `CanCreate` never read |
|
||||||
|
| 3 | **High** | security | `executeUpdate` never fires `BeforeHandle` (create/read/delete do); auth + public-rule check skipped, only `BeforeUpdate` (`CanUpdate`) runs |
|
||||||
|
| 4 | **High** | security | Create/update data keys are not validated against model columns (`q.Value(key,…)`, `SetMap(existingMap)`): mass assignment of any column, arbitrary identifiers |
|
||||||
|
| 5 | **High** | security | Row-level security (`ApplyRowSecurity`) is wired to `BeforeRead` only; update/delete by id bypass app-level RLS (DB-level RLS via `OnTxBegin` still applies) |
|
||||||
|
| 6 | **High** | security | `resolvespec_annotate` has no auth/rule check, writes through `h.db` (outside tx, no `OnTxBegin`), any `tool_name` key; annotations are agent-facing text, so it is a prompt-injection store |
|
||||||
|
| 7 | **High** | slowness | No default/max `limit`, no max `offset`, `COUNT(*)` on every read, `[]` batch create unbounded, no statement timeout |
|
||||||
|
| 8 | **Medium** | security | `dynamicSSEHandler.pool` keyed by `Host` + `X-Forwarded-Proto` (attacker-controlled): unbounded map growth and poisoned `message` endpoint URL sent to the client |
|
||||||
|
| 9 | **Medium** | security / logging | Raw `err.Error()` (DB errors, hook errors, panic value `"internal error: %s"`) returned as tool text; `logger.Error` of the same forwards to Sentry (X8) |
|
||||||
|
| 10 | **Medium** | correctness | Update reads row, merges **json-tag keys** into `SetMap` as column names, writes every column back; breaks when json tag ≠ db column, clobbers concurrent edits (no `FOR UPDATE`) |
|
||||||
|
| 11 | **Medium** | correctness | Update ignores `nil` and `""` values: a column cannot be set to NULL or empty |
|
||||||
|
| 12 | **Medium** | correctness | Create/update commit tx 1, then run tx 2 (refetch + `AfterCreate`). Tx 2 failure returns an error for a committed write; an agent retry duplicates the insert |
|
||||||
|
| 13 | **Medium** | security | Preload relation names are passed straight to `PreloadRelation` without checking the model's relations; no depth/breadth cap |
|
||||||
|
| 14 | **Low** | security | Update/delete distinguish `record not found` from hook errors, so ids can be enumerated by error text |
|
||||||
|
| 15 | **Medium** | locking | `HookRegistry.hooks` map unsynchronized; `Register`/`Clear*` race with `Execute` (same as funcspec #9) |
|
||||||
|
| 16 | **Low** | security | Filter columns are validated by `ColumnValidator` for reads only; sort/column values are interpolated unquoted after validation (relies on validator being exact); `CustomOperators`/`ComputedColumns` unreachable from tools today, keep it that way |
|
||||||
|
| 17 | **Low** | panic | `recoverPanic` returns the panic value to the client and loses the stack; hook panics in `Execute` are not recovered before the handler-level recover |
|
||||||
|
| 18 | **Low** | agent usability | Tool names embed schema+entity (`read_public_users`); no discovery tool, so clients cannot list tables without loading every tool schema |
|
||||||
|
| 19 | **Info** | testing | No tests for auth/rule enforcement, hostile filters, key validation, limits, or `-race` |
|
||||||
|
|
||||||
|
## Details
|
||||||
|
|
||||||
|
### 1. Rules invisible to hooks (High)
|
||||||
|
`NewHandlerWithGORM/Bun/DB` call `modelregistry.NewModelRegistry()`. `security` resolves rules
|
||||||
|
via `GetModelRulesFromContext` then `modelregistry.GetModelRulesByName`, which walks the
|
||||||
|
**global** list (`registries`). The handler registry is never added, so
|
||||||
|
`ErrModelNotFound` → `CheckModelUpdate/DeleteAllowed` return `nil` (allow) and
|
||||||
|
`CheckModelAuthAllowed` falls back to "auth required, public flags ignored".
|
||||||
|
`CanUpdate=false`, `CanDelete=false` are not enforced. Fix: put rules into the
|
||||||
|
request context in `withRequestData` (`security.ModelRulesKey`) and/or `AddRegistry` on
|
||||||
|
construction.
|
||||||
|
|
||||||
|
### 2-3. Create/update gating (High)
|
||||||
|
`CheckModelAuthAllowed(op)` handles public flags only. Add `BeforeCreate` →
|
||||||
|
`CheckModelCreateAllowed` (new, mirrors update/delete), and call `BeforeHandle` at the
|
||||||
|
top of `executeUpdate`.
|
||||||
|
|
||||||
|
### 4. Column allowlist (High)
|
||||||
|
Validate every key in create/update `data` against `common.NewColumnValidator(model)`;
|
||||||
|
reject unknown keys with an error (do not silently drop on writes). Also consider a
|
||||||
|
per-model writable-column set (excluding PK, `CanPublic*`-guarded columns) for agents.
|
||||||
|
|
||||||
|
### 5. RLS on writes (High)
|
||||||
|
Run `LoadSecurityRules` + a row predicate on the update/delete pre-read query; fail the
|
||||||
|
write when the row is not visible to the user.
|
||||||
|
|
||||||
|
### 6. Annotation tool (High)
|
||||||
|
Remove from default registration or gate behind `BeforeHandle` + explicit rule. Values
|
||||||
|
returned to the agent must be treated as data, not instructions.
|
||||||
|
|
||||||
|
### 7. Limits (High)
|
||||||
|
Server config: `DefaultLimit` (e.g. 50), `MaxLimit`, `MaxOffset`, `MaxBatch`, `MaxPreloadDepth`,
|
||||||
|
per-call `context.WithTimeout`. Skip `COUNT(*)` unless requested (`with_count`).
|
||||||
|
|
||||||
|
### 8. SSE pool (Medium)
|
||||||
|
Require `Config.BaseURL` for SSE, or cap/evict `pool`, and validate `Host` against an
|
||||||
|
allowlist.
|
||||||
|
|
||||||
|
### 9/17. Error surface (Medium/Low)
|
||||||
|
Map errors to stable codes + short message; log details server-side with stack
|
||||||
|
(`logger.HandlePanic`).
|
||||||
|
|
||||||
|
### 10-12. Update/create semantics (Medium)
|
||||||
|
Build the `SET` only from validated incoming keys (column names resolved from model,
|
||||||
|
not json tags); use `NULL` for explicit null; single tx including refetch and `After*`
|
||||||
|
hooks (see `audit/single_tran.md`), or return success + warning when tx 2 fails.
|
||||||
|
|
||||||
|
## Rewrite (agreed design)
|
||||||
|
|
||||||
|
Replace per-model tools with fixed meta tools. Decisions recorded 2026-09-30:
|
||||||
|
|
||||||
|
| Decision | Choice |
|
||||||
|
|---|---|
|
||||||
|
| Functions source | Explicit registry: `Handler.RegisterFunction(name, meta, fn)`; only registered functions visible/callable |
|
||||||
|
| Old tools/resources | Removed (breaking) |
|
||||||
|
| Discovery | `list_tables`, `describe_table`, `list_functions` |
|
||||||
|
| Create | `insert_into_table` added |
|
||||||
|
| Write scope | update/delete by id **or** filters; max-rows cap, `dry_run` and confirm token apply to filter writes; id writes are single-row, no token |
|
||||||
|
| Guardrails | require id/filter, max rows affected, `dry_run`, confirm token |
|
||||||
|
| Read limits / ACL | server caps (limit, offset, preload depth); list tools filtered per caller rules |
|
||||||
|
| Identity | Authenticated caller's `UserContext`; no fixed MCP user. Endpoint guarded by OAuth / session token / API key (new `resolvespec_login_api_key`); no guest mode |
|
||||||
|
| Annotations | `resolvespec_annotate` becomes opt-in (`Config.EnableAnnotations`) and goes through `BeforeHandle` |
|
||||||
|
|
||||||
|
| Tool | Purpose |
|
||||||
|
|---|---|
|
||||||
|
| `list_tables` | registered `schema.entity` visible to caller, with allowed ops |
|
||||||
|
| `describe_table` | columns, PK, relations, writable columns, rules, limits for one table |
|
||||||
|
| `select_table` | filters/sort/columns/preloads/cursor; capped |
|
||||||
|
| `insert_into_table` | one or batch (capped); column allowlist |
|
||||||
|
| `update_table` | validated keys; guardrails |
|
||||||
|
| `delete_from_table` | guardrails |
|
||||||
|
| `list_functions` | registered functions + parameter schemas |
|
||||||
|
| `call_function` | validated args, tx + hooks |
|
||||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,203 @@
|
|||||||
|
# Audit — `pkg/testmodels`
|
||||||
|
|
||||||
|
- **Date:** 2026-09-29
|
||||||
|
- **Scope:** `pkg/testmodels/business.go` (161 LOC, 1 file, **no tests**)
|
||||||
|
- **Axes:** thread locking/waiting · slowness · security · panic handling & logging
|
||||||
|
- **Threat model:** hostile internet client. These models are registered into
|
||||||
|
`pkg/modelregistry`, which means any model here becomes a reachable entity for the spec handlers.
|
||||||
|
|
||||||
|
## Summary
|
||||||
|
|
||||||
|
Six GORM struct definitions (`Department`, `Employee`, `Project`, `ProjectTask`, `Document`,
|
||||||
|
`Comment`) used as fixtures, plus two registration helpers. No concurrency, no I/O, no panics, no
|
||||||
|
logging — so three of the four audit axes are trivially clean.
|
||||||
|
|
||||||
|
Two real issues: **all six registration errors are discarded**, and this fixture package ships in
|
||||||
|
`pkg/` (not `_test.go`, not `internal/`) where a consuming application can register test tables
|
||||||
|
into a production registry.
|
||||||
|
|
||||||
|
| # | Severity | Axis | Finding |
|
||||||
|
|---|----------|------|---------|
|
||||||
|
| 1 | Medium | Correctness | `RegisterTestModels` discards all six `RegisterModel` error returns |
|
||||||
|
| 2 | Medium | Security | Fixtures live in exported `pkg/`, registerable into a production model registry |
|
||||||
|
| 3 | Low | Security | Models are registered via `RegisterModel`, which applies permissive `DefaultModelRules` |
|
||||||
|
| 4 | Low | Correctness | `GetTestModels()` return order is unrelated to FK dependency order |
|
||||||
|
| 5 | Low | Correctness | `Document.Path` is an unconstrained filesystem path exposed as a writable API field |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Findings
|
||||||
|
|
||||||
|
### 1. All registration errors discarded (Medium, Correctness)
|
||||||
|
|
||||||
|
`business.go:142-149`
|
||||||
|
|
||||||
|
```go
|
||||||
|
func RegisterTestModels(registry *modelregistry.DefaultModelRegistry) {
|
||||||
|
registry.RegisterModel("departments", Department{})
|
||||||
|
registry.RegisterModel("employees", Employee{})
|
||||||
|
registry.RegisterModel("projects", Project{})
|
||||||
|
registry.RegisterModel("project_tasks", ProjectTask{})
|
||||||
|
registry.RegisterModel("documents", Document{})
|
||||||
|
registry.RegisterModel("comments", Comment{})
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
`RegisterModel` returns `error` and every return value is dropped. The function itself returns
|
||||||
|
nothing, so a caller cannot detect failure either.
|
||||||
|
|
||||||
|
This matters more than usual because of how `pkg/modelregistry.RegisterModel` fails. It has two
|
||||||
|
error paths (`pkg/modelregistry/model_registry.go:151-158`):
|
||||||
|
|
||||||
|
```go
|
||||||
|
if !r.tryLock() {
|
||||||
|
return fmt.Errorf("failed to register model %s: registry locked", name)
|
||||||
|
}
|
||||||
|
...
|
||||||
|
if _, exists := r.models[name]; exists {
|
||||||
|
return fmt.Errorf("model %s already registered", name)
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
The first is a **transient lock-contention failure** — see `audit/pkg/modelregistry.audit.md`
|
||||||
|
finding 7, where a contended `tryLock` gives up after ~20 ms. So under concurrent registration, some
|
||||||
|
subset of these six models silently fails to register, with no error, no log, and no panic. The
|
||||||
|
process then runs with, say, `documents` and `comments` missing from the registry.
|
||||||
|
|
||||||
|
That is not merely a missing-fixture annoyance. Per `audit/pkg/modelregistry.audit.md` finding 1, an
|
||||||
|
unregistered model causes `pkg/security/hooks.go:274-294` to take the
|
||||||
|
`return nil // model not registered, allow by default` branch — so a silently-failed registration
|
||||||
|
turns into **authorisation fail-open** for that entity.
|
||||||
|
|
||||||
|
Note `errcheck` is enabled (golangci-lint v2 standard set) but `.golangci.json` excludes
|
||||||
|
`"tests?"` paths — `pkg/testmodels` does not match that pattern, so this *should* be flagged
|
||||||
|
today. Worth checking whether the linter is actually run in CI.
|
||||||
|
|
||||||
|
**Recommendation:** return `error`, and use `errors.Join` so a partial failure is reported in full:
|
||||||
|
|
||||||
|
```go
|
||||||
|
func RegisterTestModels(registry *modelregistry.DefaultModelRegistry) error {
|
||||||
|
return errors.Join(
|
||||||
|
registry.RegisterModel("departments", Department{}),
|
||||||
|
registry.RegisterModel("employees", Employee{}),
|
||||||
|
...
|
||||||
|
)
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
### 2. Fixtures are exported from `pkg/` (Medium, Security)
|
||||||
|
|
||||||
|
The package path is `github.com/bitechdev/ResolveSpec/pkg/testmodels`, not a `_test.go` file and not
|
||||||
|
under `internal/`. Consequences:
|
||||||
|
|
||||||
|
- The six structs and both helpers are part of ResolveSpec's **public API surface**. They are
|
||||||
|
compiled into every binary that imports anything which transitively imports this package.
|
||||||
|
- A consuming application (or a copy-pasted quickstart) that calls
|
||||||
|
`testmodels.RegisterTestModels(registry)` against its production registry makes
|
||||||
|
`departments`, `employees`, `projects`, `project_tasks`, `documents` and `comments` live entities
|
||||||
|
on the spec handlers, addressable by name. If the production database happens to have tables with
|
||||||
|
those names — `documents` and `comments` are very common names — the handlers will happily
|
||||||
|
read and write them under the permissive default rules of finding 3.
|
||||||
|
- It also means any future model added here for test convenience automatically becomes reachable.
|
||||||
|
|
||||||
|
Nothing in `pkg/` currently calls `RegisterTestModels` (only the test tree does), so this is a
|
||||||
|
packaging hazard rather than a live exposure.
|
||||||
|
|
||||||
|
**Recommendation:** move to `internal/testmodels` (blocks external import outright) or to a
|
||||||
|
`testmodels_test` package / `testdata` helper. If it must stay importable for downstream tests,
|
||||||
|
document loudly and consider a build tag.
|
||||||
|
|
||||||
|
### 3. Registered with permissive default rules (Low, Security)
|
||||||
|
|
||||||
|
`RegisterTestModels` uses `RegisterModel`, not `RegisterModelWithRules`. Per
|
||||||
|
`pkg/modelregistry/model_registry.go:191-194`, that initialises each model with
|
||||||
|
`DefaultModelRules()`, which grants `CanRead`, `CanUpdate`, `CanCreate` and `CanDelete` — see
|
||||||
|
`audit/pkg/modelregistry.audit.md` finding 13. `CanPublic*` are `false`, which is the saving grace.
|
||||||
|
|
||||||
|
If finding 2 is acted on this becomes moot; if these models are intended to stay registerable, they
|
||||||
|
should be registered read-only.
|
||||||
|
|
||||||
|
### 4. `GetTestModels()` order is not dependency order (Low, Correctness)
|
||||||
|
|
||||||
|
`business.go:152-160` returns the models in declaration order:
|
||||||
|
`Department, Employee, Project, ProjectTask, Document, Comment`.
|
||||||
|
|
||||||
|
The FK graph is not satisfied by that order. `Employee.DepartmentID → Department.ID` happens to work,
|
||||||
|
but `Document.OwnerID → Employee.ID` and `Document.ProjectID → Project.ID` mean `Document` must
|
||||||
|
follow both, and `ProjectTask.AssigneeID → Employee.ID` and `ProjectTask.ProjectID → Project.ID`
|
||||||
|
likewise. Coincidentally the declaration order does satisfy these — but nothing enforces it, and
|
||||||
|
`Employee.ManagerID → Employee.ID` is self-referential, which several migration/auto-migrate paths
|
||||||
|
handle only if the self-FK is deferred.
|
||||||
|
|
||||||
|
Also the two `many2many` joins (`department_projects`, `employee_projects`, declared at
|
||||||
|
`business.go:20`, `:45`, `:67-68`) are not in the returned list at all, so a caller using
|
||||||
|
`GetTestModels()` to drive `AutoMigrate` gets the join tables only because GORM infers them from the
|
||||||
|
tags — a Bun-based migration path (`pkg/common/adapters/database/bun.go`) would not.
|
||||||
|
|
||||||
|
**Recommendation:** document that the order is migration-safe and add a comment stating the
|
||||||
|
constraint, or return an explicitly ordered list with a test that asserts it.
|
||||||
|
|
||||||
|
### 5. `Document.Path` is an unconstrained path field (Low, Security)
|
||||||
|
|
||||||
|
`business.go:107`
|
||||||
|
|
||||||
|
```go
|
||||||
|
Path string `json:"path"`
|
||||||
|
```
|
||||||
|
|
||||||
|
No validation, no length limit, no `gorm` constraint. As a plain string column it is inert — the
|
||||||
|
risk only materialises if some handler or downstream consumer uses it to open a file, at which point
|
||||||
|
an attacker who can `POST`/`PATCH` a `Document` controls a filesystem path (`../../etc/passwd`,
|
||||||
|
`/proc/self/environ`). The same applies to `ContentType` (`business.go:105`) if it is ever echoed
|
||||||
|
into a response header unvalidated, and `Size` (`business.go:106`) which is a client-settable
|
||||||
|
`int64` that can disagree with reality.
|
||||||
|
|
||||||
|
Nothing in `pkg/` reads these fields, so this is a note about the fixture's shape rather than a
|
||||||
|
present vulnerability — but it is a bad example to ship, since fixtures get copied.
|
||||||
|
|
||||||
|
**Recommendation:** if these stay, mark `Path` as server-set (a `gorm:"->"` read-only tag, or
|
||||||
|
exclude it from the writable column set) so the fixture demonstrates the safe pattern.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Axis-by-axis
|
||||||
|
|
||||||
|
- **Thread locking / waiting:** nothing to report. The package declares no goroutines, channels,
|
||||||
|
mutexes or atomics. Its only concurrency exposure is *through* `pkg/modelregistry`, covered in
|
||||||
|
finding 1 and in that package's audit.
|
||||||
|
- **Slowness:** nothing to report. `RegisterTestModels` and `GetTestModels` are O(1) with six
|
||||||
|
elements and are startup-only. The `TableName()` methods (`business.go:23`, `:49`, `:73`, `:96`,
|
||||||
|
`:119`, `:137`) return constants — no allocation, no reflection.
|
||||||
|
- **Security:** findings 2, 3, 5 — all about packaging and field shape, none about code behaviour.
|
||||||
|
- **Panic handling and logging:** the package contains no `panic`, no `recover`, and does not import
|
||||||
|
`pkg/logger`. For plain struct definitions that is correct. The one place where logging *would*
|
||||||
|
belong is the discarded errors of finding 1 — silently dropping six error returns is the
|
||||||
|
panic/error-handling defect in this package, even though no panic is involved.
|
||||||
|
|
||||||
|
## What looks right
|
||||||
|
|
||||||
|
- Struct tags are consistent and complete: `json` on every field, `gorm:"primaryKey"` on every ID,
|
||||||
|
`gorm:"uniqueIndex"` on the natural keys (`Department.Code`, `Employee.Email`, `Project.Code`),
|
||||||
|
and explicit `foreignKey`/`references` on every relation rather than relying on GORM's inference.
|
||||||
|
That makes these fixtures genuinely useful for exercising the relation-expansion paths in
|
||||||
|
`pkg/restheadspec` and `pkg/resolvespec`.
|
||||||
|
- `omitempty` on every relation field prevents empty relation arrays from bloating responses — which
|
||||||
|
matters, because these fixtures are what the handler tests measure payloads against.
|
||||||
|
- Nullable FKs are correctly modelled as `*string` (`Employee.ManagerID` `business.go:35`,
|
||||||
|
`Document.ProjectID` `business.go:109`) rather than empty-string sentinels.
|
||||||
|
- The self-referential manager/reports pair (`business.go:43-44`) and the two `many2many` relations
|
||||||
|
give reasonable coverage of the harder relation shapes — a genuinely well-chosen fixture set for
|
||||||
|
the recursive-preload logic audited in `audit/pkg/restheadspec.audit.md`.
|
||||||
|
- `TableName()` is defined on the value receiver for all six, so it works whether a value or a
|
||||||
|
pointer is passed — which matters given `pkg/modelregistry.RegisterModel` normalises pointers to
|
||||||
|
values.
|
||||||
|
|
||||||
|
## Suggested follow-up
|
||||||
|
|
||||||
|
1. Return and check errors from `RegisterTestModels` (finding 1). One-line-per-call change, and it
|
||||||
|
closes a silent path to authorisation fail-open.
|
||||||
|
2. Decide whether this package belongs in `pkg/` at all (finding 2). `internal/testmodels` is the
|
||||||
|
low-effort fix.
|
||||||
|
3. Confirm `golangci-lint` runs in CI and that `errcheck` flags `business.go:143-148` — if it does
|
||||||
|
not, the exclusion patterns in `.golangci.json` need review, since this is exactly the class of
|
||||||
|
bug it exists to catch.
|
||||||
@@ -0,0 +1,286 @@
|
|||||||
|
# Audit — `pkg/tracing`
|
||||||
|
|
||||||
|
- **Date:** 2026-09-29
|
||||||
|
- **Scope:** `pkg/tracing/tracing.go` (146 LOC, 1 file, **no tests**)
|
||||||
|
- **Axes:** thread locking/waiting · slowness · security · panic handling & logging
|
||||||
|
- **Threat model:** hostile internet client. Span names and attributes here are built directly from
|
||||||
|
request-controlled data (method, path, full URL, Host header).
|
||||||
|
|
||||||
|
## Summary
|
||||||
|
|
||||||
|
A thin OpenTelemetry wrapper: `InitTracer`, an HTTP middleware, and helpers. The abstraction is
|
||||||
|
fine; the **hardcoded choices** are the problem. Three of them are not configurable at all and each
|
||||||
|
is wrong for a production, internet-facing deployment:
|
||||||
|
|
||||||
|
- `otlptracegrpc.WithInsecure()` — trace export is **plaintext**, with a source comment admitting it.
|
||||||
|
- `sdktrace.AlwaysSample()` — **100% of requests** are traced, with no sampling knob in config.
|
||||||
|
- `semconv.HTTPURLKey.String(r.URL.String())` — the **full URL including query string** is exported.
|
||||||
|
|
||||||
|
Combined: every request's full URL is shipped unencrypted to a collector, and an attacker sets the
|
||||||
|
export volume. Span names are also built from raw paths, giving unbounded cardinality.
|
||||||
|
|
||||||
|
`config.TracingConfig` (`pkg/config/config.go:86-91`) exposes only `Enabled`, `ServiceName`,
|
||||||
|
`ServiceVersion` and `Endpoint` — there is no field for TLS or sample rate, so these cannot be fixed
|
||||||
|
by configuration alone.
|
||||||
|
|
||||||
|
| # | Severity | Axis | Finding |
|
||||||
|
|---|----------|------|---------|
|
||||||
|
| 1 | **High** | Security | `WithInsecure()` hardcoded — traces exported in plaintext, not configurable |
|
||||||
|
| 2 | **High** | Security | Full URL **including query string** exported as a span attribute |
|
||||||
|
| 3 | **High** | Slowness | `AlwaysSample()` hardcoded — 100% trace volume, attacker-controlled, no sampling config |
|
||||||
|
| 4 | Medium | Slowness | Span name is `method + " " + r.URL.Path` — unbounded cardinality from raw path IDs |
|
||||||
|
| 5 | Medium | Locking | `tracer` global written by `InitTracer`, read unsynchronised by `Middleware`/`StartSpan` |
|
||||||
|
| 6 | Medium | Observability | `Middleware` records no HTTP status and no error status — spans never show failures |
|
||||||
|
| 7 | Medium | Panic | `Middleware` does not recover; a downstream panic leaves the span unmarked (`Unset` status) |
|
||||||
|
| 8 | Low | Slowness | `InitTracer` has no timeout/deadline on exporter or resource creation |
|
||||||
|
| 9 | Low | Maintenance | `semconv/v1.4.0` (2021) — deprecated attribute names modern collectors no longer index |
|
||||||
|
| 10 | Low | Security | `SetAttributes`/`AddEvent` pass caller data through with no size or cardinality limit |
|
||||||
|
|
||||||
|
## Resolution (2026-09-30)
|
||||||
|
|
||||||
|
Fixed in `pkg/tracing/tracing.go`, `pkg/config` (`TracingConfig`, defaults), the package README, and new
|
||||||
|
`pkg/tracing/tracing_test.go` (passes).
|
||||||
|
|
||||||
|
| # | Status | What changed |
|
||||||
|
|---|--------|--------------|
|
||||||
|
| 1 | **Fixed** | TLS is the default. `Config` gains `Insecure`, `TLSConfig` and `Headers` (OTLP auth). `tracing.insecure` added to `pkg/config`. **Breaking:** plaintext collectors now need `Insecure: true`. |
|
||||||
|
| 2 | **Fixed** | Query string and `Host` are no longer exported; attributes are method, `url.path`, scheme, `http.route`, status. `TLSConfig` has no config-file key (code only). |
|
||||||
|
| 3 | **Fixed** | `ParentBased(TraceIDRatioBased(rate))`; `SampleRate` defaults to 0.1, validated to [0,1]; `tracing.sample_rate` added to `pkg/config`. |
|
||||||
|
| 4 | **Fixed** | Span name is `METHOD <route template>` from `Request.Pattern`, `<unmatched>` otherwise. `MiddlewareWithRoute(fn)` supports other routers. |
|
||||||
|
| 5 | **Fixed** | `tracer` is an `atomic.Pointer`; a second `InitTracer` returns an error; the shutdown func resets state. |
|
||||||
|
| 6 | **Fixed** | Response writer wrapped; `http.response.status_code` recorded, 5xx sets Error status. Preserves `Flush`/`Unwrap`. |
|
||||||
|
| 7 | **Fixed** | Panics are recorded (`RecordError`, Error status) and re-raised so the panic middleware still responds. Must be installed inside the panic middleware; actual order in `pkg/server` not verified. |
|
||||||
|
| 8 | **Fixed** | `InitTracerContext(ctx, cfg)` with `InitTimeout` (default 10s); `InitTracer` retained as a wrapper. Exporter is shut down if resource creation fails. |
|
||||||
|
| 9 | **Fixed** | Moved to `semconv/v1.26.0`. |
|
||||||
|
| 10 | **Fixed** | `AttributeValueLengthLimit` set via `WithRawSpanLimits` (`AttributeValueLimit`, default 1024). |
|
||||||
|
|
||||||
|
Tests added: query redaction and route naming, unmatched route, 5xx status, panic recorded and re-raised,
|
||||||
|
double-init and invalid sample rate.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Findings
|
||||||
|
|
||||||
|
### 1. `WithInsecure()` hardcoded (High, Security)
|
||||||
|
|
||||||
|
`tracing.go:38-42`
|
||||||
|
|
||||||
|
```go
|
||||||
|
client := otlptracegrpc.NewClient(
|
||||||
|
otlptracegrpc.WithEndpoint(config.Endpoint),
|
||||||
|
otlptracegrpc.WithInsecure(), // Use WithTLSCredentials in production
|
||||||
|
)
|
||||||
|
```
|
||||||
|
|
||||||
|
The comment names the fix and the code does not implement it, and — critically — `Config`
|
||||||
|
(`tracing.go:21-27`) has no field to express it:
|
||||||
|
|
||||||
|
```go
|
||||||
|
type Config struct {
|
||||||
|
ServiceName string
|
||||||
|
ServiceVersion string
|
||||||
|
Endpoint string
|
||||||
|
Enabled bool
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
So there is **no supported way** to enable TLS on trace export short of editing this file. Every
|
||||||
|
span — carrying the full request URL per finding 2 — crosses the network in cleartext, and the
|
||||||
|
collector endpoint is unauthenticated (no OTLP headers/bearer token option either), so anything that
|
||||||
|
can reach it can also *inject* fabricated spans.
|
||||||
|
|
||||||
|
**Recommendation:** add `Insecure bool`, `TLSConfig *tls.Config` and `Headers map[string]string` to
|
||||||
|
`Config` (and the matching `tracing.*` keys to `pkg/config`), default to TLS on, and require an
|
||||||
|
explicit opt-in for insecure. Wire `otlptracegrpc.WithTLSCredentials` / `WithHeaders`.
|
||||||
|
|
||||||
|
### 2. Full URL with query string exported (High, Security)
|
||||||
|
|
||||||
|
`tracing.go:95-103`
|
||||||
|
|
||||||
|
```go
|
||||||
|
ctx, span := tracer.Start(ctx, r.Method+" "+r.URL.Path,
|
||||||
|
trace.WithSpanKind(trace.SpanKindServer),
|
||||||
|
trace.WithAttributes(
|
||||||
|
semconv.HTTPMethodKey.String(r.Method),
|
||||||
|
semconv.HTTPURLKey.String(r.URL.String()), // <- full URL, query string included
|
||||||
|
semconv.HTTPTargetKey.String(r.URL.Path),
|
||||||
|
semconv.HTTPSchemeKey.String(r.URL.Scheme),
|
||||||
|
semconv.NetHostNameKey.String(r.Host),
|
||||||
|
),
|
||||||
|
)
|
||||||
|
```
|
||||||
|
|
||||||
|
`r.URL.String()` includes `RawQuery`. For this API the query string is where the interesting data
|
||||||
|
lives: filter expressions, column lists, and — for any client that passes credentials as a query
|
||||||
|
parameter (`?api_key=`, `?token=`, signed-URL style parameters) — secrets. All of it lands in the
|
||||||
|
tracing backend, and per finding 1 it gets there in plaintext.
|
||||||
|
|
||||||
|
Note `HTTPTargetKey` is also set to `r.URL.Path`, so the *useful* part is already captured
|
||||||
|
separately; `HTTPURLKey` adds only the sensitive part.
|
||||||
|
|
||||||
|
Secondary: `r.Host` comes from the `Host` header, which is client-controlled and unvalidated here —
|
||||||
|
so an attacker can pollute the `net.host.name` dimension with arbitrary values (cardinality blowup,
|
||||||
|
and log/dashboard spoofing).
|
||||||
|
|
||||||
|
**Recommendation:** export a redacted URL (scheme + host + path, query keys only or dropped
|
||||||
|
entirely). OTel's own guidance is to strip or redact query parameters for exactly this reason.
|
||||||
|
|
||||||
|
### 3. `AlwaysSample()` hardcoded (High, Slowness)
|
||||||
|
|
||||||
|
`tracing.go:61-65`
|
||||||
|
|
||||||
|
```go
|
||||||
|
tp := sdktrace.NewTracerProvider(
|
||||||
|
sdktrace.WithBatcher(exporter),
|
||||||
|
sdktrace.WithResource(res),
|
||||||
|
sdktrace.WithSampler(sdktrace.AlwaysSample()),
|
||||||
|
)
|
||||||
|
```
|
||||||
|
|
||||||
|
Every request produces a recorded, exported span. There is no `SampleRate` in `Config` and no
|
||||||
|
`tracing.sample_rate` key in `pkg/config/manager.go`'s defaults, so this is not tunable.
|
||||||
|
|
||||||
|
Under the hostile-client threat model the request rate — and therefore the span rate, the batch
|
||||||
|
queue pressure, the serialisation cost and the outbound bandwidth — is set by the attacker. Each
|
||||||
|
request pays span allocation, attribute encoding (including the full URL string), and a share of
|
||||||
|
batch export. When the batch queue fills, the SDK drops spans, so a flood also destroys the
|
||||||
|
observability you need to see the flood.
|
||||||
|
|
||||||
|
**Recommendation:** default to `sdktrace.ParentBased(sdktrace.TraceIDRatioBased(rate))` with a
|
||||||
|
configurable rate (e.g. 0.01–0.1), keeping `AlwaysSample` available for development. `ParentBased`
|
||||||
|
also means an upstream sampling decision is respected, which `AlwaysSample` currently overrides.
|
||||||
|
|
||||||
|
### 4. Unbounded span-name cardinality (Medium, Slowness)
|
||||||
|
|
||||||
|
`tracing.go:95` — the span name is `r.Method + " " + r.URL.Path`.
|
||||||
|
|
||||||
|
This API's paths embed identifiers (`/api/public/employees/7f3c…`, `/api/<schema>/<entity>/<id>`), so
|
||||||
|
each distinct ID becomes a distinct span name. Consequences:
|
||||||
|
|
||||||
|
- Tracing backends index on span name; unbounded distinct names is the classic cardinality-explosion
|
||||||
|
cost bomb (and in some backends, a hard limit that starts rejecting data).
|
||||||
|
- It violates the OTel HTTP convention, which requires a **low-cardinality route template**
|
||||||
|
(`GET /api/{schema}/{entity}/{id}`), with the concrete value in `http.route`/attributes.
|
||||||
|
- It is attacker-driven: requests to random paths — including 404s — each mint a new span name.
|
||||||
|
|
||||||
|
**Recommendation:** derive the name from the matched route pattern. `pkg/server`'s router
|
||||||
|
(chi/mux/gin, see `audit/pkg/server.audit.md`) exposes the route template after matching; use it, and
|
||||||
|
place this middleware after the router so the pattern is available. Fall back to
|
||||||
|
`r.Method + " " + "<unmatched>"` rather than the raw path.
|
||||||
|
|
||||||
|
### 5. Unsynchronised `tracer` global (Medium, Locking)
|
||||||
|
|
||||||
|
`tracing.go:19`
|
||||||
|
|
||||||
|
```go
|
||||||
|
var tracer trace.Tracer
|
||||||
|
```
|
||||||
|
|
||||||
|
Written at `tracing.go:77` (`tracer = tp.Tracer(config.ServiceName)`), read at `tracing.go:86`,
|
||||||
|
`:95` (`Middleware`) and `:116`, `:119` (`StartSpan`). No mutex, no `atomic.Value`.
|
||||||
|
|
||||||
|
Same pattern as `pkg/logger`'s `Logger` global (see `audit/pkg/logger.audit.md` finding 1). Benign if
|
||||||
|
`InitTracer` runs once before any request is served; a race the moment tracing is re-initialised at
|
||||||
|
runtime. `InitTracer` is exported and callable at any time, and calling it twice also leaks the
|
||||||
|
first `TracerProvider` (nothing shuts it down) — its batch processor goroutine and gRPC connection
|
||||||
|
stay alive for the life of the process.
|
||||||
|
|
||||||
|
Note the nil checks at `:86` and `:116` are the read-half of the race: a goroutine can observe a
|
||||||
|
non-nil-but-torn interface value.
|
||||||
|
|
||||||
|
**Recommendation:** `atomic.Pointer` or a `sync.Once`-guarded init; return an error from a second
|
||||||
|
`InitTracer` call, or shut down the previous provider first.
|
||||||
|
|
||||||
|
### 6. No HTTP status or error status on spans (Medium, Observability)
|
||||||
|
|
||||||
|
`Middleware` (`tracing.go:84-112`) never wraps `w`, so it cannot observe the status code. It sets no
|
||||||
|
`semconv.HTTPStatusCodeKey` and never calls `span.SetStatus`. Every span therefore has status
|
||||||
|
`Unset`, which tracing backends render as "OK".
|
||||||
|
|
||||||
|
The practical effect: you cannot find failing requests in the traces. A 500-storm and a healthy
|
||||||
|
period look identical in the span data, which defeats the main reason to run tracing on an
|
||||||
|
internet-facing service.
|
||||||
|
|
||||||
|
**Recommendation:** wrap the `ResponseWriter` to capture the status, set
|
||||||
|
`semconv.HTTPStatusCodeKey.Int(status)`, and `span.SetStatus(codes.Error, ...)` for 5xx.
|
||||||
|
|
||||||
|
### 7. No panic handling in the middleware (Medium, Panic handling)
|
||||||
|
|
||||||
|
`Middleware` has `defer span.End()` (`tracing.go:106`) but no `recover()`. If `next.ServeHTTP`
|
||||||
|
panics:
|
||||||
|
|
||||||
|
- The `defer span.End()` **does** run, so no span is leaked — that part is correct.
|
||||||
|
- But the span is ended with status `Unset` and no exception event, so the panic is invisible in the
|
||||||
|
trace. The one place a trace would be most valuable records nothing.
|
||||||
|
- The panic propagates up to whichever handler is outermost. Whether that is
|
||||||
|
`pkg/middleware/panic.go` depends on middleware ordering — if `tracing.Middleware` is installed
|
||||||
|
*outside* the panic middleware, the panic escapes to `net/http`'s per-connection recovery, which
|
||||||
|
kills the connection and logs to the default logger, bypassing `pkg/logger` and the error tracker
|
||||||
|
entirely. See `audit/pkg/middleware.audit.md` and `audit/pkg/server.audit.md` for the actual order.
|
||||||
|
|
||||||
|
This package does not import `pkg/logger` at all, so nothing here can be logged.
|
||||||
|
|
||||||
|
**Recommendation:** recover, record `span.RecordError` + `span.SetStatus(codes.Error, …)`, then
|
||||||
|
re-panic so the dedicated panic middleware still handles the response. Document the required
|
||||||
|
middleware order.
|
||||||
|
|
||||||
|
### 8. No deadline on initialisation (Low, Slowness)
|
||||||
|
|
||||||
|
`tracing.go:36` uses `ctx := context.Background()` for both `otlptrace.New` (`:44`) and
|
||||||
|
`resource.New` (`:51`). `otlptracegrpc` does not block on connect by default, so this is unlikely to
|
||||||
|
hang today — but `resource.New` with detectors can perform network calls (cloud metadata endpoints),
|
||||||
|
and an unreachable metadata service is a classic multi-second startup stall. `InitTracer` should
|
||||||
|
accept a `context.Context` from the caller so startup has a deadline.
|
||||||
|
|
||||||
|
### 9. `semconv/v1.4.0` (Low, Maintenance)
|
||||||
|
|
||||||
|
`tracing.go:15` pins the 2021 semantic conventions. `http.method`, `http.url`, `http.target`,
|
||||||
|
`http.scheme`, `net.host.name` were all renamed in v1.20+ (`http.request.method`, `url.full`,
|
||||||
|
`url.path`, `url.scheme`, `server.address`). Current collectors, dashboards and backend
|
||||||
|
auto-instrumentation views key off the new names, so these spans will not populate standard HTTP
|
||||||
|
dashboards.
|
||||||
|
|
||||||
|
### 10. No limits on caller-supplied span data (Low, Security)
|
||||||
|
|
||||||
|
`StartSpan`, `AddEvent`, `SetAttributes` (`tracing.go:115-145`) forward caller attributes verbatim.
|
||||||
|
If any caller passes request-derived values (a filter expression, a row payload), span size is
|
||||||
|
attacker-influenced. The SDK's default limits (128 attributes, 128 events) cap the count but not the
|
||||||
|
*value* length — a 1 MB string attribute is accepted.
|
||||||
|
|
||||||
|
**Recommendation:** set explicit `sdktrace.WithSpanLimits` including `AttributeValueLengthLimit`.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## What looks right
|
||||||
|
|
||||||
|
- **Disabled path is genuinely free.** `InitTracer` with `Enabled: false` (`tracing.go:31-34`)
|
||||||
|
returns a no-op shutdown func and never builds an exporter, so a disabled deployment pays nothing
|
||||||
|
and cannot leak.
|
||||||
|
- **Nil-tracer guards everywhere.** `Middleware` (`tracing.go:86-89`) passes through untouched and
|
||||||
|
`StartSpan` (`tracing.go:116-118`) returns the incoming context plus the context's (no-op) span.
|
||||||
|
So a partially-initialised process degrades safely rather than nil-panicking — a pattern
|
||||||
|
`pkg/logger` gets right too.
|
||||||
|
- **Context propagation is correct.** `Extract` from `propagation.HeaderCarrier(r.Header)`
|
||||||
|
(`tracing.go:92`), a composite `TraceContext` + `Baggage` propagator (`tracing.go:72-75`), and
|
||||||
|
`r = r.WithContext(ctx)` (`tracing.go:109`) before calling `next` — the span context actually
|
||||||
|
reaches downstream handlers, which is the part most hand-rolled middlewares get wrong.
|
||||||
|
- `SpanKindServer` is set correctly (`tracing.go:96`).
|
||||||
|
- `WithBatcher` rather than a simple/sync span processor (`tracing.go:62`) — export does not block
|
||||||
|
the request path.
|
||||||
|
- `InitTracer` returns `tp.Shutdown` (`tracing.go:80`), giving the caller a real flush-on-shutdown
|
||||||
|
hook with a caller-supplied context, which is better than the fixed-timeout pattern in
|
||||||
|
`pkg/errortracking` (see that audit, finding 3).
|
||||||
|
- `RecordError` nil-guards (`tracing.go:140-143`) so `RecordError(ctx, nil)` is a no-op.
|
||||||
|
|
||||||
|
## Suggested follow-up
|
||||||
|
|
||||||
|
1. Extend `Config` with `Insecure`, TLS credentials, OTLP headers and `SampleRate`; add the matching
|
||||||
|
keys to `pkg/config` (findings 1, 3). These cannot be fixed without an API change, so they should
|
||||||
|
go together.
|
||||||
|
2. Redact the query string from exported attributes (finding 2).
|
||||||
|
3. Move to route-template span names, which requires positioning the middleware after routing
|
||||||
|
(finding 4).
|
||||||
|
4. Capture status code and panics in the middleware (findings 6, 7).
|
||||||
|
5. Guard the `tracer` global (finding 5) and upgrade `semconv` (finding 9).
|
||||||
|
6. Add tests: this package has none. A tracetest/in-memory exporter makes assertions on span name,
|
||||||
|
attributes and status straightforward, and would have caught findings 2, 4 and 6.
|
||||||
@@ -0,0 +1,122 @@
|
|||||||
|
# Single transaction per request — plan
|
||||||
|
|
||||||
|
## Goal
|
||||||
|
- Every DB statement and every hook that touches the DB in one request runs on **one transaction / one connection**.
|
||||||
|
- Hooks never receive the raw pool (`h.db`).
|
||||||
|
- Fixes: RLS GUCs (`set_config(..., true)`) lost on reads/creates/deletes; extra pool connections; select-then-write races.
|
||||||
|
|
||||||
|
## Why
|
||||||
|
- `set_config(..., true)` is transaction-local. A hook on the pool, or a query on another pool connection, never sees it → RLS returns 0 rows / 42501.
|
||||||
|
- Each un-transacted call takes its own pool connection → bursts with a small pool (see `dbtrace`).
|
||||||
|
- Already fixed: read/create hooks in `resolvespec` + `restheadspec` (commit `47708fc`, tag >= v1.1.28). Consumers on older tags still show the bug.
|
||||||
|
|
||||||
|
## Current state (verified by reading code; not yet by `dbtrace`)
|
||||||
|
| Spec | Read | Create | Update | Delete |
|
||||||
|
|---|---|---|---|---|
|
||||||
|
| restheadspec | tx; `AfterRead` post-commit on pool | tx; `AfterCreate` post-commit on pool | tx; re-fetch + `BeforeScan` post-commit on pool (`:1667-1674`) | **single: no tx, hook + select + delete on pool (`:1945-1994`)**; batch: tx, per-item `BeforeDelete` inside |
|
||||||
|
| resolvespec | tx | tx | tx; re-fetch on pool (`:1297, 1449, 1602`) | **single: hook + select + delete on pool (`:1654, 1794, 1806`)**; batch: one `BeforeDelete` before tx, none per item |
|
||||||
|
| websocketspec | **pool** (`:563-672`) | **pool** (`:708`) | **pool** (`:744`) | **pool** (`:757`) |
|
||||||
|
| mqttspec | **pool** (`:674-789`) | **pool** (`:838`) | **pool** (`:875`) | **pool** (`:889`) |
|
||||||
|
| resolvemcp | **pool** (`:253`) | single: **pool** (`:445`); batch: tx | tx | tx |
|
||||||
|
| funcspec | tx; `BeforeResponse` post-commit on pool (`:337, 640`) | — | — | — |
|
||||||
|
|
||||||
|
- Correction to earlier note: "no transactions" in mqttspec/websocketspec/resolvemcp-read is a gap for this problem, not a non-issue.
|
||||||
|
- `BeforeHandle` runs before any tx by design (auth + model checks, `PreloadSecurityRules`). Keep it DB-free except security preload (own connection, cached).
|
||||||
|
|
||||||
|
## In-tx hook coverage today (verified)
|
||||||
|
- Already in tx with `Tx: tx`: `BeforeRead`, `BeforeCreate`, `BeforeUpdate`, `BeforeScan` (read/create/update, both specs); restheadspec batch delete `BeforeDelete` + `AfterDelete`.
|
||||||
|
- **Not in tx:** single `BeforeDelete`/`AfterDelete` (both specs), resolvespec batch delete (no per-item hook), all `After*` post-commit, all websocketspec/mqttspec hooks, resolvemcp read/single create.
|
||||||
|
- Gap beyond coverage: no single guaranteed "tx opened" point. User/RLS stamping would have to be repeated in each `Before*` hook and is missed by any path without one (e.g. resolvespec batch delete). `OnTxBegin` closes this: fires once per tx, first, for read/insert/update/delete and for the second short tx.
|
||||||
|
|
||||||
|
## Scope
|
||||||
|
- `OnTxBegin` + `runInTx` apply to **all six**: resolvespec, restheadspec, websocketspec, mqttspec, resolvemcp, funcspec.
|
||||||
|
- Each spec has its own `HookType` (resolvespec, restheadspec, websocketspec, resolvemcp, funcspec); mqttspec aliases websocketspec, so it inherits the constant but needs its own handler wiring.
|
||||||
|
- Same semantics everywhere: fires once per tx, first, for read/insert/update/delete and the second short tx; failure aborts + rolls back, nothing leaked.
|
||||||
|
- Each spec's `security_hooks.go` registers the user/RLS stamping on `OnTxBegin`.
|
||||||
|
- Shared helper preferred over six copies: one small function in `pkg/common` (begin tx, set `Tx`, call spec-supplied begin callback), each spec passes its own hook executor.
|
||||||
|
|
||||||
|
## funcspec (different)
|
||||||
|
- Custom SQL handlers (`SqlQuery`, `SqlQueryList`), no CRUD, no model registry; one tx per request already (`:195`, `:561`).
|
||||||
|
- Already in tx: `BeforeQuery`/`BeforeQueryList`, `BeforeSQLExec`, `AfterSQLExec`, `AfterQuery`/`AfterQueryList`, plus `BeforeOp`.
|
||||||
|
- `BeforeOp` = generic pre-hook via `ExecuteBeforeOp`, fires before every `Before*` in the tx; but it fires **twice** per tx (query hook + `BeforeSQLExec`), so it is not a once-per-tx point.
|
||||||
|
- Only gap: `BeforeResponse` runs post-commit with `Tx = h.db` (`:337`, `:640`).
|
||||||
|
- Applies from this plan: once-per-tx `OnTxBegin` (stamping user/RLS before any SQL, incl. hook-mutated SQL), `BeforeResponse` on a tx, fail-closed abort.
|
||||||
|
- Does not apply: delete/insert/update phases, second re-fetch tx (no re-fetch; SQL is user-defined), `BeforeHandle` preload.
|
||||||
|
- Decided: add a real `OnTxBegin`. `BeforeOp` unchanged.
|
||||||
|
|
||||||
|
## Common interface (`pkg/common`, new `txhook.go`)
|
||||||
|
- Precedent: `security.SecurityContext` + per-spec `newSecurityContext(hookCtx)` adapter. Same pattern here.
|
||||||
|
- `common.TxHookName` = `"on_tx_begin"`: one shared string; each spec declares `OnTxBegin HookType = common.TxHookName` (HookTypes are per-spec types, so the constant value is shared, not the type).
|
||||||
|
- `common.TxContext` interface, implemented by each spec's `HookContext` via a small adapter: `GetContext()`, `GetTx()`, `SetTx(common.Database)`, plus `Abort` accessors for the abort path.
|
||||||
|
- `common.RunRequestTx(ctx, db, tc TxContext, onBegin func() error, body func(tx common.Database) error) error`: `RunInTransaction` -> `tc.SetTx(tx)` -> `onBegin()` (spec passes `registry.Execute(OnTxBegin, hookCtx)`) -> `body(tx)`. `onBegin` error or abort = return error = rollback.
|
||||||
|
- Shared stamping: one function in `pkg/security` taking `SecurityContext` + `common.Database` (sets tx-local user/RLS); each spec's `RegisterSecurityHooks` registers it on `OnTxBegin`. No per-spec copies of the logic.
|
||||||
|
- Each spec keeps its own registry/`HookContext`; only the tx lifecycle and stamping are shared.
|
||||||
|
- Out of scope: unifying the six `HookContext` / registry types.
|
||||||
|
|
||||||
|
## Design
|
||||||
|
1. **`OnTxBegin` hook** (new `HookType`, all specs). Runs first inside every tx the handler opens; gets `tx` in `hookCtx.Tx`. RLS stamping is registered once there; reads owner/tenant from request context.
|
||||||
|
2. **`runInTx` helper per handler**: wraps `RunInTransaction`, sets `hookCtx.Tx = tx`, fires `OnTxBegin`, runs the body. All handler paths use it; no path passes `h.db` to a hook.
|
||||||
|
3. **Post-commit work** (`After*`, update re-fetch, `BeforeResponse`): run in a second short `runInTx` (so `OnTxBegin` re-applies). Not inside the main tx.
|
||||||
|
4. **Delete**: hook → select → delete in one tx; 404 on no row; cache invalidation after commit.
|
||||||
|
5. **Backward compat**: hooks keep the same names/order; only `hookCtx.Tx` changes from pool to tx. `OnTxBegin` is additive.
|
||||||
|
|
||||||
|
## Decisions (settled)
|
||||||
|
- Insert/update: re-fetch + `AfterCreate`/`AfterUpdate`-style post-commit work run in a **second short tx** (must see trigger changes). Only insert/update; read and delete have no second tx.
|
||||||
|
- Update re-fetch is a plain SELECT in that second tx. No `RETURNING`.
|
||||||
|
- `OnTxBegin` failure aborts the whole request, rolls back, returns an error with no detail leaked to the client.
|
||||||
|
|
||||||
|
## Open
|
||||||
|
- Consumer's ResolveSpec version: confirm it is >= v1.1.28 (read/create already in tx). Not blocking.
|
||||||
|
|
||||||
|
## Phases
|
||||||
|
| # | Status | Change | Files | Notes |
|
||||||
|
|---|---|---|---|---|
|
||||||
|
| 0 | DONE | Baseline: enable `dbtrace` on testserver, record `tx/tx_queries/pooled/raw` per op | `cmd/testserver`, `pkg/dbtrace` | pooled > 0 on write ops = the gaps above |
|
||||||
|
| 1 | DONE | Delete in one tx (single + batch, per-item hooks inside tx) | `resolvespec/handler.go`, `restheadspec/handler.go` | fixes 2 pool connections + race + RLS |
|
||||||
|
| 2 | DONE | `OnTxBegin` hook type + `runInTx` helper | `common/txhook.go`, `*/hooks.go`, `*/handler.go` | resolvespec + restheadspec; other specs in P4-6 |
|
||||||
|
| 3 | DONE | Insert/update post-commit hooks + re-fetch in second short `runInTx` (select only) | restheadspec `:1005, 1467, 1667-1674`; resolvespec `:1297, 1449, 1602` | per decision above |
|
||||||
|
| 4 | DONE | websocketspec + mqttspec: wrap read/create/update/delete in `runInTx` | `websocketspec/handler.go`, `mqttspec/handler.go` | mqttspec aliases websocketspec hooks; confirm `OnTxBegin` alias |
|
||||||
|
| 5 | DONE | resolvemcp: read + single create in tx | `resolvemcp/handler.go:253, 445` | verify batch/update/delete hooks run inside tx |
|
||||||
|
| 6 | DONE | funcspec: `OnTxBegin` (or once-per-tx `BeforeOp`), `BeforeResponse` via `runInTx` | `funcspec/function_api.go:337, 640` | |
|
||||||
|
| 7 | DONE | Security hooks: register RLS stamping on `OnTxBegin`; document | `pkg/security/*`, README | |
|
||||||
|
|
||||||
|
## Progress
|
||||||
|
- DONE P0: baseline via `dbtrace` on real Postgres (commit `cd96404`): create/read/delete `pooled=0`; update `pooled=1` (re-fetch) = P3 target. websocketspec/mqttspec/resolvemcp not measured.
|
||||||
|
- DONE P1: single + batch delete in one tx (resolvespec, restheadspec). Not done: per-item `BeforeDelete` in resolvespec batch (behavior change, deferred).
|
||||||
|
- DONE infra: `sqlmock` delete tx tests (both specs); compose test server + `scripts/testserver-smoke.sh` (podman first); testmodels ids now serial.
|
||||||
|
- NOTE: restheadspec single delete still does the lookup before `BeforeDelete`; safe once `OnTxBegin` (P2) exists. An `AfterDelete` failure now rolls the delete back.
|
||||||
|
- DONE P2 (resolvespec + restheadspec): `common.TxHookName`, `common.TxContext` (`SetTx` only; no abort/context accessors needed since `Execute` already returns an error on abort), `common.RunRequestTx`; per-spec `OnTxBegin`, `HookContext.SetTx`, `Handler.runInTx`. Every `RunInTransaction` in both handlers now goes through it. Tests: `pkg/*/on_tx_begin_test.go` (once, first, on tx, failure rolls back). Not yet: the post-commit second tx (P3) and the security stamping registration (P7).
|
||||||
|
- DONE P3: restheadspec update re-fetch + `BeforeScan` + `AfterUpdate` and `AfterCreate` run in a second short `runInTx`; resolvespec update re-fetches (single, both batch paths) run in a second short `runInTx`. Fixed the pool reads inside the first tx (resolvespec single/batch update existing-record select, restheadspec update existence select) to use `tx`. Tests: `pkg/*/update_tx_test.go` (restheadspec uses the bun adapter; the pgsql adapter cannot build model-based updates).
|
||||||
|
- NOTE: resolvespec fires no `AfterCreate`/`AfterRead`/`AfterUpdate`-post-commit hooks other than `AfterUpdate` inside the tx; nothing more to move there.
|
||||||
|
- OPEN: restheadspec `AfterRead` still runs post-commit with `Tx = h.db` (`:1004`); decision says read has no second tx. Needs a call: run it inside the read tx, or in a short second tx.
|
||||||
|
- DONE P4: websocketspec + mqttspec. `OnTxBegin` (mqttspec re-exports the websocketspec constant), `HookContext.SetTx`, per-handler `runInTx`/`sendTxError`. Per message: read = 1 tx (Before/After hooks + queries); delete = 1 tx (Before, delete, After); create/update = tx 1 (Before + write) then tx 2 (re-fetch + `BeforeScan` + After). `create()`/`update()` no longer re-fetch; `read*`/`create`/`update`/`delete` use `hookCtx.Tx`. websocketspec `FetchRowNumber` keeps its public signature and delegates to a new tx-aware `fetchRowNumber`. A failure in begin/`OnTxBegin`/commit answers `transaction_error` with no detail. Tests: `pkg/websocketspec/tx_test.go` (sqlmock), `pkg/mqttspec/tx_test.go` (sqlite); mqttspec `update` tests now pass `Tx`.
|
||||||
|
- DECIDED in P4 (follow `AfterRead` question above): websocketspec/mqttspec run `AfterRead` inside the read tx (keeps "read has no second tx").
|
||||||
|
- DONE P5: resolvemcp. `OnTxBegin`, `HookContext.SetTx`, `Handler.runInTx`. Read = 1 tx (`BeforeRead`, count, scan, `AfterRead`; `readInTx`). Delete = 1 tx (`BeforeDelete` moved inside, after `OnTxBegin`). Create (single and batch, unified) = tx 1 (`BeforeCreate` + inserts) then tx 2 (re-fetch + `AfterCreate`); the old single-record pool insert/re-fetch is gone. Update = tx 1 (select, `BeforeUpdate`, update, `AfterUpdate`) then tx 2 (re-fetch). `BeforeHandle` still runs before any tx with `Tx = h.db`. Tests: `pkg/resolvemcp/tx_test.go` (sqlmock).
|
||||||
|
- DONE P6: funcspec. `OnTxBegin`, `HookContext.SetTx`, `Handler.runInTx` for `SqlQuery` and `SqlQueryList`. `BeforeResponse` now runs in a second short tx (`Tx` is no longer the pool). `BeforeOp` is unchanged (still per statement). A begin/`OnTxBegin`/commit failure answers 500 `transaction_error` / "Transaction failed" (before, it returned with no response); body failures still answer via `sendError`. Tests: `pkg/funcspec/tx_test.go`.
|
||||||
|
- DONE (AfterRead, decided by user): restheadspec `AfterRead` now runs in a second short tx. Test: `pkg/restheadspec/read_tx_test.go`.
|
||||||
|
- DONE P7: `pkg/security/txsettings.go`: `SecurityList.SetTxSettings(fn)`, `StampTxSettings`, `ApplyTxSettings` (configurable map, decided by user; `set_config(name, value, true)`, value hex-encoded, name validated, Postgres only, fail closed). Every spec's `RegisterSecurityHooks` registers it on `OnTxBegin`. Tests: `pkg/security/txsettings_test.go`, `pkg/resolvespec/tx_settings_test.go`. Docs: `pkg/common/TRANSACTIONS.md`.
|
||||||
|
- DONE real-Postgres check (resolvespec, testserver via compose): create `tx=1 pooled=0`, read `tx=1 pooled=0`, update `tx=2 pooled=0` (was `pooled=1`), single delete `tx=1 pooled=0`, batch create/delete `tx=1 pooled=0`. Compose now uses host networking (bridge fails here): testserver on 8123, Postgres on 8124 (was 8080/5434); integration test DSNs updated. Smoke script covers read and update. websocketspec/mqttspec/resolvemcp/restheadspec/funcspec not measured on real Postgres.
|
||||||
|
- DONE regression tests: per-spec read/create/update/delete hook-on-tx, failure-rollback (Before*/After*/`OnTxBegin`) and second-tx tests in all six specs (`ops_tx_test.go`, `tx_test.go`, `read_tx_test.go`); stamping tests for resolvespec, resolvemcp, funcspec; pgsql adapter preload tests (same connection, error returned); source guard `pkg/common/tx_guard_test.go` (no direct `RunInTransaction`/`BeginTx`, no `Tx = h.db` beyond the allowlisted BeforeHandle placeholders, no pool statements in spec handlers).
|
||||||
|
- FIXED: resolvespec now fires `AfterRead` (in the read tx; `Result` = scanned slice for single and list) `AfterCreate` (in the create tx, per record, all four create paths) and `AfterDelete` (in the delete tx, once per request; a failure rolls the delete back). Before, neither fired, so `AfterRead` column-level security masking (registered by `RegisterSecurityHooks`) was silently skipped on resolvespec reads. Failing `AfterRead`/`AfterCreate` fails the request and rolls back. Tests: `pkg/resolvespec/ops_tx_test.go` (incl. end-to-end column hiding).
|
||||||
|
- FIXED: restheadspec total-count cache key now includes the record id; a read by id (total 1) no longer poisons the list total for the 2-minute TTL. Tests: `pkg/restheadspec/cache_key_test.go`. resolvespec is unaffected (its count runs before the id filter, so the total is the list total by design). The cache is still process-wide, so read tests call `resetTotalCache`.
|
||||||
|
- NOTE: `pkg/security` `TestDatabaseAuthenticator` fails with `-count=2` (also on `cd96404`, before this work); use `-count=1`.
|
||||||
|
|
||||||
|
## Tests
|
||||||
|
- Existing: per-spec `handler_test.go`, `hooks_test.go`, `integration_test.go`; models in `pkg/testmodels/business.go`; `dbtrace` unit tests.
|
||||||
|
- Done: delete tx tests (`pkg/*/delete_tx_test.go`, sqlmock, 1-conn pool detects pool use). Missing: same for read/create/update, `OnTxBegin`, other specs.
|
||||||
|
- Add per spec/op: hook `Tx` is not the pool; `OnTxBegin` fires once per tx, before other hooks; single-ID delete = 1 tx; `dbtrace` `pooled == 0` on the request path.
|
||||||
|
- Test data: reuse `pkg/testmodels`; **ask before generating new data** (per project rule).
|
||||||
|
- Regression: full `go test -race` for security, dbmanager, common, restheadspec, resolvespec, websocketspec, mqttspec, resolvemcp, funcspec. Known pre-existing failures: mqttspec integration (no DB).
|
||||||
|
|
||||||
|
## Risks
|
||||||
|
- Long tx if a hook does slow work inside it → hold connection longer; keep hooks fast.
|
||||||
|
- Pool of 1: nothing inside a tx may take a second pool connection (auth/security loads are outside; keep it so).
|
||||||
|
- Behavior change: After hooks no longer get the pool handle; hooks that relied on an independent connection break.
|
||||||
|
- websocket/mqtt long-lived connections: tx must be per message, never per connection.
|
||||||
|
|
||||||
|
## Done when
|
||||||
|
- `dbtrace` shows `pooled=0` for every handler op on a hooked model.
|
||||||
|
- RLS GUC set in `OnTxBegin` is visible to read, create, update, delete queries and hooks.
|
||||||
|
- No `Tx: h.db` / `hookCtx.Tx = h.db` left in spec handlers.
|
||||||
|
- OPEN: websocketspec `BeforeDisconnect`/`AfterDisconnect` are defined but never executed (connection lifecycle, not DB). Allowlisted in `TestEveryDefinedHookHasACallSite`; wire them to remove the entry.
|
||||||
|
- DONE: column-level hide/mask columns are dropped from create/update payloads (`security.ApplyWriteColumnSecurity`); rules preloaded in `BeforeHandle` for create/update. resolvemcp update now runs `BeforeHandle`.
|
||||||
@@ -0,0 +1,12 @@
|
|||||||
|
# Clients
|
||||||
|
|
||||||
|
| Dir | Language | Specs | Verified |
|
||||||
|
|---|---|---|---|
|
||||||
|
| `resolvespec-js` | TypeScript | ResolveSpec, HeaderSpec, FunctionSpec, WebSocketSpec | yes |
|
||||||
|
| `resolvespec-python` | Python >= 3.11 | ResolveSpec, HeaderSpec, FunctionSpec, WebSocketSpec | yes (61 tests) |
|
||||||
|
| `resolvespec-go` | Go | ResolveSpec, FunctionSpec | yes (`go test`) |
|
||||||
|
| `resolvespec-rs` | Rust | ResolveSpec, FunctionSpec | yes (`cargo test`) |
|
||||||
|
| `resolvespec-cs` | C# (.NET 8) | ResolveSpec, FunctionSpec | yes (`dotnet test`) |
|
||||||
|
| `resolvespec-dart` | Dart / Flutter | ResolveSpec, FunctionSpec | yes (`dart test`) |
|
||||||
|
|
||||||
|
Wire behaviour is identical across clients; FunctionSpec server quirks are listed in each README.
|
||||||
@@ -0,0 +1,2 @@
|
|||||||
|
bin/
|
||||||
|
obj/
|
||||||
@@ -0,0 +1,43 @@
|
|||||||
|
# ResolveSpec.Client (C#)
|
||||||
|
|
||||||
|
.NET 8 client for ResolveSpec (JSON body) and FunctionSpec. `System.Text.Json`, no other dependencies.
|
||||||
|
|
||||||
|
> Tests run with `DOTNET_ROLL_FORWARD=Major` when only a newer runtime than 8.0 is installed.
|
||||||
|
|
||||||
|
## Clients
|
||||||
|
|
||||||
|
| Type | Constructor | Methods |
|
||||||
|
|---|---|---|
|
||||||
|
| `ResolveSpecClient` | `(baseUrl, ClientOptions?)` | `GetMetadataAsync` `ReadAsync` `CreateAsync` `UpdateAsync` `DeleteAsync` |
|
||||||
|
| `FuncSpecClient` | `(baseUrl, ClientOptions?)` | `QueryAsync` `QueryListAsync` |
|
||||||
|
|
||||||
|
`ClientOptions`: `Token`, `Headers`, `Timeout`, `HttpClient`. Precedence: Content-Type < custom headers < bearer token.
|
||||||
|
|
||||||
|
## ResolveSpec
|
||||||
|
|
||||||
|
- `id`: int/long/string → URL, `IEnumerable<string>` → body.
|
||||||
|
- `Options` with nullable properties; wire names via `JsonPropertyName`.
|
||||||
|
- Result: `Response{Success, Data (JsonElement), Metadata}`; `resp.Decode<T>()`.
|
||||||
|
|
||||||
|
## FunctionSpec
|
||||||
|
|
||||||
|
- Routes are server-defined: pass the `path`.
|
||||||
|
- Params (`IDictionary<string, object?>`) → query string (enumerable → repeated keys, bool → `true`/`false`, null skipped).
|
||||||
|
- `FuncSpecOptions` → `X-*` headers: `Filters`, `SearchFilters`, `CustomSqlWhere`, `CustomSqlOr`, `Sort`, `Limit`, `Offset`, `Distinct`, `SkipCount`, `SkipCache`, `ResponseFormat`.
|
||||||
|
- `QueryListAsync` fills `Metadata` from `Content-Range`; 206 is success.
|
||||||
|
- Static helpers: `BuildHeaders`, `BuildQuery`, `EncodeHeaderValue`, `DecodeHeaderValue`.
|
||||||
|
|
||||||
|
## Server quirks
|
||||||
|
|
||||||
|
- `Sort` is raw SQL in ORDER BY (client sends `col ASC|DESC`).
|
||||||
|
- One search operator per column.
|
||||||
|
- Values starting `ZIP_` / `__` are base64-decoded by the server.
|
||||||
|
- Non-ASCII, control chars and edge spaces are auto-encoded (`ZIP_`).
|
||||||
|
|
||||||
|
## Errors
|
||||||
|
|
||||||
|
`ResolveSpecException{StatusCode, Message, Error{Code, Detail, Sql}}`.
|
||||||
|
|
||||||
|
## Test
|
||||||
|
|
||||||
|
`dotnet test tests/`
|
||||||
@@ -0,0 +1,187 @@
|
|||||||
|
using System.Globalization;
|
||||||
|
using System.Text;
|
||||||
|
using System.Text.Json;
|
||||||
|
using System.Text.RegularExpressions;
|
||||||
|
|
||||||
|
namespace ResolveSpec;
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Options sent to funcspec endpoints as X-* headers.
|
||||||
|
/// Server behaviour (pkg/funcspec): Sort is inserted raw into ORDER BY (so it is sent as SQL terms);
|
||||||
|
/// only one search operator per column is kept; values starting with "ZIP_" or "__" are
|
||||||
|
/// base64-decoded by the server, so such plaintext values cannot be sent faithfully.
|
||||||
|
/// </summary>
|
||||||
|
public sealed class FuncSpecOptions
|
||||||
|
{
|
||||||
|
/// <summary>eq+AND -> X-FieldFilter; others X-SearchOp / X-SearchOr.</summary>
|
||||||
|
public List<FilterOption>? Filters { get; set; }
|
||||||
|
/// <summary>X-SearchFilter-{col}: text ILIKE.</summary>
|
||||||
|
public Dictionary<string, string>? SearchFilters { get; set; }
|
||||||
|
public string? CustomSqlWhere { get; set; }
|
||||||
|
public string? CustomSqlOr { get; set; }
|
||||||
|
public List<SortOption>? Sort { get; set; }
|
||||||
|
public int? Limit { get; set; }
|
||||||
|
public int? Offset { get; set; }
|
||||||
|
public bool? Distinct { get; set; }
|
||||||
|
public bool? SkipCount { get; set; }
|
||||||
|
public bool? SkipCache { get; set; }
|
||||||
|
/// <summary>simple | detail | syncfusion</summary>
|
||||||
|
public string? ResponseFormat { get; set; }
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>Client for user-defined SQL endpoints. Routes are defined by the server application.</summary>
|
||||||
|
public sealed class FuncSpecClient
|
||||||
|
{
|
||||||
|
readonly Transport _t;
|
||||||
|
|
||||||
|
public FuncSpecClient(string baseUrl, ClientOptions? options = null) => _t = new Transport(baseUrl, options);
|
||||||
|
|
||||||
|
static readonly Dictionary<string, string> OperatorMap = new()
|
||||||
|
{
|
||||||
|
["eq"] = "equals", ["neq"] = "notequals", ["gt"] = "greaterthan", ["gte"] = "greaterthanorequal",
|
||||||
|
["lt"] = "lessthan", ["lte"] = "lessthanorequal", ["like"] = "contains", ["ilike"] = "contains",
|
||||||
|
["contains"] = "contains", ["startswith"] = "beginswith", ["endswith"] = "endswith", ["in"] = "in",
|
||||||
|
["between"] = "between", ["between_inclusive"] = "betweeninclusive",
|
||||||
|
["is_null"] = "empty", ["is_not_null"] = "notempty",
|
||||||
|
};
|
||||||
|
|
||||||
|
static string Scalar(object? v) => v switch
|
||||||
|
{
|
||||||
|
null => "",
|
||||||
|
string s => s,
|
||||||
|
bool b => b ? "true" : "false",
|
||||||
|
JsonElement { ValueKind: JsonValueKind.Null } => "",
|
||||||
|
JsonElement e => e.ValueKind == JsonValueKind.String ? e.GetString() ?? "" : e.ToString(),
|
||||||
|
IFormattable f => f.ToString(null, CultureInfo.InvariantCulture),
|
||||||
|
_ => v.ToString() ?? "",
|
||||||
|
};
|
||||||
|
|
||||||
|
static string FilterValue(object? v) =>
|
||||||
|
v is System.Collections.IEnumerable list and not string
|
||||||
|
? string.Join(",", list.Cast<object?>().Select(Scalar))
|
||||||
|
: Scalar(v);
|
||||||
|
|
||||||
|
/// <summary>Base64 (UTF-8) with the ZIP_ prefix.</summary>
|
||||||
|
public static string EncodeHeaderValue(string v) => "ZIP_" + Convert.ToBase64String(Encoding.UTF8.GetBytes(v));
|
||||||
|
|
||||||
|
/// <summary>Decode a value that may carry a ZIP_ or __ prefix (nested allowed).</summary>
|
||||||
|
public static string DecodeHeaderValue(string v)
|
||||||
|
{
|
||||||
|
foreach (var p in new[] { "ZIP_", "__" })
|
||||||
|
{
|
||||||
|
if (!v.StartsWith(p, StringComparison.Ordinal)) continue;
|
||||||
|
var b64 = Regex.Replace(v[p.Length..], "[\n\r ]", "");
|
||||||
|
b64 = b64.PadRight(b64.Length + (4 - b64.Length % 4) % 4, '=');
|
||||||
|
try { return DecodeHeaderValue(Encoding.UTF8.GetString(Convert.FromBase64String(b64))); }
|
||||||
|
catch (FormatException) { return v; }
|
||||||
|
}
|
||||||
|
return v;
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>Encode values that are unsafe as raw header/query text (non-ASCII, control chars, edge spaces).</summary>
|
||||||
|
static string Safe(string v) =>
|
||||||
|
v != v.Trim() || v.Any(c => c > 127 || char.IsControl(c)) ? EncodeHeaderValue(v) : v;
|
||||||
|
|
||||||
|
/// <summary>Build the X-* headers understood by funcspec.ParseParameters.</summary>
|
||||||
|
public static Dictionary<string, string> BuildHeaders(FuncSpecOptions? o)
|
||||||
|
{
|
||||||
|
var h = new Dictionary<string, string>();
|
||||||
|
if (o == null) return h;
|
||||||
|
|
||||||
|
foreach (var f in o.Filters ?? new())
|
||||||
|
{
|
||||||
|
var logic = string.IsNullOrEmpty(f.LogicOperator) ? "AND" : f.LogicOperator;
|
||||||
|
var v = Safe(FilterValue(f.Value));
|
||||||
|
if (f.Operator == "eq" && logic == "AND") { h[$"X-FieldFilter-{f.Column}"] = v; continue; }
|
||||||
|
var op = OperatorMap.TryGetValue(f.Operator, out var m) ? m : f.Operator;
|
||||||
|
h[$"{(logic == "OR" ? "X-SearchOr" : "X-SearchOp")}-{op}-{f.Column}"] = v;
|
||||||
|
}
|
||||||
|
foreach (var (col, text) in o.SearchFilters ?? new()) h[$"X-SearchFilter-{col}"] = Safe(text);
|
||||||
|
if (!string.IsNullOrEmpty(o.CustomSqlWhere)) h["X-Custom-SQL-W"] = Safe(o.CustomSqlWhere);
|
||||||
|
if (!string.IsNullOrEmpty(o.CustomSqlOr)) h["X-Custom-SQL-Or"] = Safe(o.CustomSqlOr);
|
||||||
|
if (o.Sort is { Count: > 0 })
|
||||||
|
{
|
||||||
|
// funcspec puts this verbatim into ORDER BY
|
||||||
|
h["X-Sort"] = Safe(string.Join(",", o.Sort.Select(s =>
|
||||||
|
$"{s.Column} {(string.Equals(s.Direction, "desc", StringComparison.OrdinalIgnoreCase) ? "DESC" : "ASC")}")));
|
||||||
|
}
|
||||||
|
if (o.Limit != null) h["X-Limit"] = o.Limit.Value.ToString(CultureInfo.InvariantCulture);
|
||||||
|
if (o.Offset != null) h["X-Offset"] = o.Offset.Value.ToString(CultureInfo.InvariantCulture);
|
||||||
|
if (o.Distinct != null) h["X-Distinct"] = Bool(o.Distinct.Value);
|
||||||
|
if (o.SkipCount != null) h["X-SkipCount"] = Bool(o.SkipCount.Value);
|
||||||
|
if (o.SkipCache != null) h["X-SkipCache"] = Bool(o.SkipCache.Value);
|
||||||
|
switch (o.ResponseFormat)
|
||||||
|
{
|
||||||
|
case "simple": h["X-SimpleApi"] = "true"; break;
|
||||||
|
case "detail": h["X-DetailApi"] = "true"; break;
|
||||||
|
case "syncfusion": h["X-Syncfusion"] = "true"; break;
|
||||||
|
}
|
||||||
|
return h;
|
||||||
|
}
|
||||||
|
|
||||||
|
static string Bool(bool b) => b ? "true" : "false";
|
||||||
|
|
||||||
|
/// <summary>Build query-string pairs: bools -> true/false, lists -> repeated keys, null skipped.</summary>
|
||||||
|
public static List<KeyValuePair<string, string>> BuildQuery(IDictionary<string, object?>? p)
|
||||||
|
{
|
||||||
|
var o = new List<KeyValuePair<string, string>>();
|
||||||
|
foreach (var (k, v) in p ?? new Dictionary<string, object?>())
|
||||||
|
{
|
||||||
|
if (v == null) continue;
|
||||||
|
if (v is System.Collections.IEnumerable list and not string)
|
||||||
|
foreach (var e in list) o.Add(new(k, Safe(Scalar(e))));
|
||||||
|
else o.Add(new(k, Safe(Scalar(v))));
|
||||||
|
}
|
||||||
|
return o;
|
||||||
|
}
|
||||||
|
|
||||||
|
static readonly Regex ContentRange = new(@"(\d+)-(\d+)/(\d+)");
|
||||||
|
|
||||||
|
static Metadata MetadataFrom(string? contentRange, FuncSpecOptions? o)
|
||||||
|
{
|
||||||
|
var m = new Metadata { Limit = o?.Limit ?? 0 };
|
||||||
|
var g = ContentRange.Match(contentRange ?? "");
|
||||||
|
if (g.Success)
|
||||||
|
{
|
||||||
|
var start = long.Parse(g.Groups[1].Value, CultureInfo.InvariantCulture);
|
||||||
|
var end = long.Parse(g.Groups[2].Value, CultureInfo.InvariantCulture);
|
||||||
|
var total = long.Parse(g.Groups[3].Value, CultureInfo.InvariantCulture);
|
||||||
|
m.Total = total; m.Filtered = total; m.Count = end - start; m.Offset = start;
|
||||||
|
}
|
||||||
|
return m;
|
||||||
|
}
|
||||||
|
|
||||||
|
async Task<Response> CallAsync(HttpMethod method, string path, IDictionary<string, object?>? p, FuncSpecOptions? o, bool list, CancellationToken ct)
|
||||||
|
{
|
||||||
|
var url = $"{_t.BaseUrl}/{path.TrimStart('/')}";
|
||||||
|
var q = BuildQuery(p);
|
||||||
|
if (q.Count > 0)
|
||||||
|
url += "?" + string.Join("&", q.Select(kv => $"{Uri.EscapeDataString(kv.Key)}={Uri.EscapeDataString(kv.Value)}"));
|
||||||
|
|
||||||
|
var (resp, text) = await _t.SendAsync(method, url, null, BuildHeaders(o), ct).ConfigureAwait(false);
|
||||||
|
var status = (int)resp.StatusCode;
|
||||||
|
if (!resp.IsSuccessStatusCode) throw Transport.ErrorFrom(status, text, resp.ReasonPhrase); // 206 is success
|
||||||
|
|
||||||
|
var r = new Response
|
||||||
|
{
|
||||||
|
Success = true,
|
||||||
|
Data = string.IsNullOrWhiteSpace(text) ? JsonDocument.Parse("null").RootElement.Clone() : JsonDocument.Parse(text).RootElement.Clone(),
|
||||||
|
};
|
||||||
|
if (list)
|
||||||
|
{
|
||||||
|
// Content-Range is a content header in HttpClient; fall back to response headers.
|
||||||
|
IEnumerable<string>? cr = null;
|
||||||
|
if (!resp.Content.Headers.TryGetValues("Content-Range", out cr)) resp.Headers.TryGetValues("Content-Range", out cr);
|
||||||
|
r.Metadata = MetadataFrom(cr?.FirstOrDefault(), o);
|
||||||
|
}
|
||||||
|
return r;
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>Single-record endpoint (SqlQuery). Data is the row object.</summary>
|
||||||
|
public Task<Response> QueryAsync(string path, IDictionary<string, object?>? p = null, FuncSpecOptions? o = null, HttpMethod? method = null, CancellationToken ct = default) =>
|
||||||
|
CallAsync(method ?? HttpMethod.Get, path, p, o, false, ct);
|
||||||
|
|
||||||
|
/// <summary>List endpoint (SqlQueryList). Metadata comes from Content-Range.</summary>
|
||||||
|
public Task<Response> QueryListAsync(string path, IDictionary<string, object?>? p = null, FuncSpecOptions? o = null, HttpMethod? method = null, CancellationToken ct = default) =>
|
||||||
|
CallAsync(method ?? HttpMethod.Get, path, p, o, true, ct);
|
||||||
|
}
|
||||||
@@ -0,0 +1,78 @@
|
|||||||
|
using System.Net.Http.Headers;
|
||||||
|
using System.Text;
|
||||||
|
using System.Text.Json;
|
||||||
|
|
||||||
|
namespace ResolveSpec;
|
||||||
|
|
||||||
|
/// <summary>Shared HTTP configuration for both clients.</summary>
|
||||||
|
public sealed class ClientOptions
|
||||||
|
{
|
||||||
|
public string? Token { get; set; }
|
||||||
|
public Dictionary<string, string> Headers { get; } = new(StringComparer.OrdinalIgnoreCase);
|
||||||
|
public TimeSpan Timeout { get; set; } = TimeSpan.FromSeconds(30);
|
||||||
|
/// <summary>Supply your own HttpClient (tests, pooling). Its BaseAddress is ignored.</summary>
|
||||||
|
public HttpClient? HttpClient { get; set; }
|
||||||
|
}
|
||||||
|
|
||||||
|
internal sealed class Transport
|
||||||
|
{
|
||||||
|
public readonly string BaseUrl;
|
||||||
|
readonly ClientOptions _o;
|
||||||
|
readonly HttpClient _http;
|
||||||
|
|
||||||
|
public Transport(string baseUrl, ClientOptions? o)
|
||||||
|
{
|
||||||
|
BaseUrl = baseUrl.TrimEnd('/');
|
||||||
|
_o = o ?? new ClientOptions();
|
||||||
|
_http = _o.HttpClient ?? new HttpClient { Timeout = _o.Timeout };
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>Content-Type < custom headers < per-call headers < bearer token.</summary>
|
||||||
|
public async Task<(HttpResponseMessage resp, string body)> SendAsync(
|
||||||
|
HttpMethod method, string url, string? json, IDictionary<string, string>? extra, CancellationToken ct)
|
||||||
|
{
|
||||||
|
using var req = new HttpRequestMessage(method, url);
|
||||||
|
if (json != null) req.Content = new StringContent(json, Encoding.UTF8, "application/json");
|
||||||
|
foreach (var (k, v) in _o.Headers) Set(req, k, v);
|
||||||
|
if (extra != null) foreach (var (k, v) in extra) Set(req, k, v);
|
||||||
|
if (!string.IsNullOrEmpty(_o.Token)) req.Headers.Authorization = new AuthenticationHeaderValue("Bearer", _o.Token);
|
||||||
|
var resp = await _http.SendAsync(req, ct).ConfigureAwait(false);
|
||||||
|
var body = await resp.Content.ReadAsStringAsync(ct).ConfigureAwait(false);
|
||||||
|
return (resp, body);
|
||||||
|
}
|
||||||
|
|
||||||
|
static void Set(HttpRequestMessage req, string name, string value)
|
||||||
|
{
|
||||||
|
req.Headers.Remove(name);
|
||||||
|
if (!req.Headers.TryAddWithoutValidation(name, value) && req.Content != null)
|
||||||
|
{
|
||||||
|
req.Content.Headers.Remove(name);
|
||||||
|
req.Content.Headers.TryAddWithoutValidation(name, value);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public static ResolveSpecException ErrorFrom(int status, string body, string? reason)
|
||||||
|
{
|
||||||
|
ApiError? err = null;
|
||||||
|
var isJson = false;
|
||||||
|
try
|
||||||
|
{
|
||||||
|
using var doc = JsonDocument.Parse(body);
|
||||||
|
isJson = true;
|
||||||
|
if (doc.RootElement.ValueKind == JsonValueKind.Object && doc.RootElement.TryGetProperty("error", out var e) && e.ValueKind == JsonValueKind.Object)
|
||||||
|
err = e.Deserialize<ApiError>();
|
||||||
|
}
|
||||||
|
catch (JsonException) { }
|
||||||
|
|
||||||
|
var message = err?.Message;
|
||||||
|
if (string.IsNullOrEmpty(message))
|
||||||
|
{
|
||||||
|
var text = isJson ? "" : body.Trim();
|
||||||
|
if (text.Length > 200) text = text[..200];
|
||||||
|
message = text.Length > 0 ? text : $"{reason ?? "Error"} ({status})";
|
||||||
|
}
|
||||||
|
return new ResolveSpecException(message, status, err);
|
||||||
|
}
|
||||||
|
|
||||||
|
public static string Segment(string s) => Uri.EscapeDataString(s);
|
||||||
|
}
|
||||||
@@ -0,0 +1,11 @@
|
|||||||
|
<Project Sdk="Microsoft.NET.Sdk">
|
||||||
|
<PropertyGroup>
|
||||||
|
<TargetFramework>net8.0</TargetFramework>
|
||||||
|
<Nullable>enable</Nullable>
|
||||||
|
<ImplicitUsings>enable</ImplicitUsings>
|
||||||
|
<RootNamespace>ResolveSpec</RootNamespace>
|
||||||
|
<PackageId>ResolveSpec.Client</PackageId>
|
||||||
|
<Version>0.1.0</Version>
|
||||||
|
<Description>Client for ResolveSpec (JSON body) and FunctionSpec endpoints</Description>
|
||||||
|
</PropertyGroup>
|
||||||
|
</Project>
|
||||||
@@ -0,0 +1,65 @@
|
|||||||
|
using System.Text.Json;
|
||||||
|
using System.Text.Json.Serialization;
|
||||||
|
|
||||||
|
namespace ResolveSpec;
|
||||||
|
|
||||||
|
/// <summary>Client for the ResolveSpec JSON body protocol: POST {operation, data, options}.</summary>
|
||||||
|
public sealed class ResolveSpecClient
|
||||||
|
{
|
||||||
|
static readonly JsonSerializerOptions Json = new() { DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull };
|
||||||
|
readonly Transport _t;
|
||||||
|
|
||||||
|
public ResolveSpecClient(string baseUrl, ClientOptions? options = null) => _t = new Transport(baseUrl, options);
|
||||||
|
|
||||||
|
sealed class Request
|
||||||
|
{
|
||||||
|
[JsonPropertyName("operation")] public string Operation { get; set; } = "";
|
||||||
|
[JsonPropertyName("id")] public string[]? Id { get; set; }
|
||||||
|
[JsonPropertyName("data")] public object? Data { get; set; }
|
||||||
|
[JsonPropertyName("options")] public Options? Options { get; set; }
|
||||||
|
}
|
||||||
|
|
||||||
|
// A single id (int/long/string) goes in the URL; string[] / IEnumerable<string> goes in the body.
|
||||||
|
static string? UrlId(object? id) => id switch
|
||||||
|
{
|
||||||
|
null => null,
|
||||||
|
string s => s,
|
||||||
|
IEnumerable<string> => null,
|
||||||
|
_ => Convert.ToString(id, System.Globalization.CultureInfo.InvariantCulture),
|
||||||
|
};
|
||||||
|
|
||||||
|
static string[]? BodyId(object? id) => id is IEnumerable<string> e ? e.ToArray() : null;
|
||||||
|
|
||||||
|
string Url(string schema, string entity, string? id)
|
||||||
|
{
|
||||||
|
var u = $"{_t.BaseUrl}/{Transport.Segment(schema)}/{Transport.Segment(entity)}";
|
||||||
|
return string.IsNullOrEmpty(id) ? u : $"{u}/{Transport.Segment(id)}";
|
||||||
|
}
|
||||||
|
|
||||||
|
async Task<Response> SendAsync(HttpMethod method, string url, Request? body, CancellationToken ct)
|
||||||
|
{
|
||||||
|
var json = body == null ? null : JsonSerializer.Serialize(body, Json);
|
||||||
|
var (resp, text) = await _t.SendAsync(method, url, json, null, ct).ConfigureAwait(false);
|
||||||
|
var status = (int)resp.StatusCode;
|
||||||
|
if (!resp.IsSuccessStatusCode) throw Transport.ErrorFrom(status, text, resp.ReasonPhrase);
|
||||||
|
var r = JsonSerializer.Deserialize<Response>(text, Json) ?? new Response();
|
||||||
|
if (!r.Success && r.Error != null) throw new ResolveSpecException(r.Error.Message, status, r.Error);
|
||||||
|
return r;
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>GET /{schema}/{entity}</summary>
|
||||||
|
public Task<Response> GetMetadataAsync(string schema, string entity, CancellationToken ct = default) =>
|
||||||
|
SendAsync(HttpMethod.Get, Url(schema, entity, null), null, ct);
|
||||||
|
|
||||||
|
public Task<Response> ReadAsync(string schema, string entity, object? id = null, Options? options = null, CancellationToken ct = default) =>
|
||||||
|
SendAsync(HttpMethod.Post, Url(schema, entity, UrlId(id)), new Request { Operation = "read", Id = BodyId(id), Options = options }, ct);
|
||||||
|
|
||||||
|
public Task<Response> CreateAsync(string schema, string entity, object data, Options? options = null, CancellationToken ct = default) =>
|
||||||
|
SendAsync(HttpMethod.Post, Url(schema, entity, null), new Request { Operation = "create", Data = data, Options = options }, ct);
|
||||||
|
|
||||||
|
public Task<Response> UpdateAsync(string schema, string entity, object data, object? id = null, Options? options = null, CancellationToken ct = default) =>
|
||||||
|
SendAsync(HttpMethod.Post, Url(schema, entity, UrlId(id)), new Request { Operation = "update", Id = BodyId(id), Data = data, Options = options }, ct);
|
||||||
|
|
||||||
|
public Task<Response> DeleteAsync(string schema, string entity, object id, CancellationToken ct = default) =>
|
||||||
|
SendAsync(HttpMethod.Post, Url(schema, entity, UrlId(id)), new Request { Operation = "delete" }, ct);
|
||||||
|
}
|
||||||
@@ -0,0 +1,137 @@
|
|||||||
|
using System.Text.Json;
|
||||||
|
using System.Text.Json.Serialization;
|
||||||
|
|
||||||
|
namespace ResolveSpec;
|
||||||
|
|
||||||
|
// Types aligned with Go pkg/common/types.go. JsonPropertyName values are the wire names.
|
||||||
|
|
||||||
|
public sealed class FilterOption
|
||||||
|
{
|
||||||
|
[JsonPropertyName("column")] public string Column { get; set; } = "";
|
||||||
|
/// <summary>eq neq gt gte lt lte like ilike in contains startswith endswith between between_inclusive is_null is_not_null</summary>
|
||||||
|
[JsonPropertyName("operator")] public string Operator { get; set; } = "eq";
|
||||||
|
[JsonPropertyName("value")] public object? Value { get; set; }
|
||||||
|
/// <summary>AND | OR</summary>
|
||||||
|
[JsonPropertyName("logic_operator")] public string? LogicOperator { get; set; }
|
||||||
|
}
|
||||||
|
|
||||||
|
public sealed class SortOption
|
||||||
|
{
|
||||||
|
[JsonPropertyName("column")] public string Column { get; set; } = "";
|
||||||
|
/// <summary>asc | desc</summary>
|
||||||
|
[JsonPropertyName("direction")] public string Direction { get; set; } = "asc";
|
||||||
|
}
|
||||||
|
|
||||||
|
public sealed class Parameter
|
||||||
|
{
|
||||||
|
[JsonPropertyName("name")] public string Name { get; set; } = "";
|
||||||
|
[JsonPropertyName("value")] public string Value { get; set; } = "";
|
||||||
|
[JsonPropertyName("sequence")] public int? Sequence { get; set; }
|
||||||
|
}
|
||||||
|
|
||||||
|
public sealed class CustomOperator
|
||||||
|
{
|
||||||
|
[JsonPropertyName("name")] public string Name { get; set; } = "";
|
||||||
|
[JsonPropertyName("sql")] public string Sql { get; set; } = "";
|
||||||
|
}
|
||||||
|
|
||||||
|
public sealed class ComputedColumn
|
||||||
|
{
|
||||||
|
[JsonPropertyName("name")] public string Name { get; set; } = "";
|
||||||
|
[JsonPropertyName("expression")] public string Expression { get; set; } = "";
|
||||||
|
}
|
||||||
|
|
||||||
|
public sealed class PreloadOption
|
||||||
|
{
|
||||||
|
[JsonPropertyName("relation")] public string? Relation { get; set; }
|
||||||
|
[JsonPropertyName("table_name")] public string? TableName { get; set; }
|
||||||
|
[JsonPropertyName("columns")] public List<string>? Columns { get; set; }
|
||||||
|
[JsonPropertyName("omit_columns")] public List<string>? OmitColumns { get; set; }
|
||||||
|
[JsonPropertyName("sort")] public List<SortOption>? Sort { get; set; }
|
||||||
|
[JsonPropertyName("filters")] public List<FilterOption>? Filters { get; set; }
|
||||||
|
[JsonPropertyName("where")] public string? Where { get; set; }
|
||||||
|
[JsonPropertyName("limit")] public int? Limit { get; set; }
|
||||||
|
[JsonPropertyName("offset")] public int? Offset { get; set; }
|
||||||
|
[JsonPropertyName("updateable")] public bool? Updateable { get; set; }
|
||||||
|
[JsonPropertyName("computed_ql")] public Dictionary<string, string>? ComputedQl { get; set; }
|
||||||
|
[JsonPropertyName("recursive")] public bool? Recursive { get; set; }
|
||||||
|
[JsonPropertyName("primary_key")] public string? PrimaryKey { get; set; }
|
||||||
|
[JsonPropertyName("related_key")] public string? RelatedKey { get; set; }
|
||||||
|
[JsonPropertyName("foreign_key")] public string? ForeignKey { get; set; }
|
||||||
|
[JsonPropertyName("recursive_child_key")] public string? RecursiveChildKey { get; set; }
|
||||||
|
[JsonPropertyName("sql_joins")] public List<string>? SqlJoins { get; set; }
|
||||||
|
[JsonPropertyName("join_aliases")] public List<string>? JoinAliases { get; set; }
|
||||||
|
}
|
||||||
|
|
||||||
|
public sealed class VectorSearchOption
|
||||||
|
{
|
||||||
|
[JsonPropertyName("column")] public string Column { get; set; } = "";
|
||||||
|
[JsonPropertyName("vector")] public List<double> Vector { get; set; } = new();
|
||||||
|
/// <summary>l2 (default) | cosine | ip</summary>
|
||||||
|
[JsonPropertyName("metric")] public string? Metric { get; set; }
|
||||||
|
/// <summary>Distance column alias, default _distance.</summary>
|
||||||
|
[JsonPropertyName("as")] public string? As { get; set; }
|
||||||
|
[JsonPropertyName("direction")] public string? Direction { get; set; }
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>ResolveSpec request options object.</summary>
|
||||||
|
public sealed class Options
|
||||||
|
{
|
||||||
|
[JsonPropertyName("preload")] public List<PreloadOption>? Preload { get; set; }
|
||||||
|
[JsonPropertyName("columns")] public List<string>? Columns { get; set; }
|
||||||
|
[JsonPropertyName("omit_columns")] public List<string>? OmitColumns { get; set; }
|
||||||
|
[JsonPropertyName("filters")] public List<FilterOption>? Filters { get; set; }
|
||||||
|
[JsonPropertyName("sort")] public List<SortOption>? Sort { get; set; }
|
||||||
|
[JsonPropertyName("limit")] public int? Limit { get; set; }
|
||||||
|
[JsonPropertyName("offset")] public int? Offset { get; set; }
|
||||||
|
[JsonPropertyName("customOperators")] public List<CustomOperator>? CustomOperators { get; set; }
|
||||||
|
[JsonPropertyName("computedColumns")] public List<ComputedColumn>? ComputedColumns { get; set; }
|
||||||
|
[JsonPropertyName("parameters")] public List<Parameter>? Parameters { get; set; }
|
||||||
|
[JsonPropertyName("cursor_forward")] public string? CursorForward { get; set; }
|
||||||
|
[JsonPropertyName("cursor_backward")] public string? CursorBackward { get; set; }
|
||||||
|
[JsonPropertyName("fetch_row_number")] public string? FetchRowNumber { get; set; }
|
||||||
|
[JsonPropertyName("vector_search")] public VectorSearchOption? VectorSearch { get; set; }
|
||||||
|
}
|
||||||
|
|
||||||
|
public sealed class Metadata
|
||||||
|
{
|
||||||
|
[JsonPropertyName("total")] public long Total { get; set; }
|
||||||
|
[JsonPropertyName("count")] public long Count { get; set; }
|
||||||
|
[JsonPropertyName("filtered")] public long Filtered { get; set; }
|
||||||
|
[JsonPropertyName("limit")] public long Limit { get; set; }
|
||||||
|
[JsonPropertyName("offset")] public long Offset { get; set; }
|
||||||
|
}
|
||||||
|
|
||||||
|
public sealed class ApiError
|
||||||
|
{
|
||||||
|
[JsonPropertyName("code")] public string Code { get; set; } = "";
|
||||||
|
[JsonPropertyName("message")] public string Message { get; set; } = "";
|
||||||
|
[JsonPropertyName("details")] public JsonElement? Details { get; set; }
|
||||||
|
/// <summary>Server-side reason (funcspec / restheadspec).</summary>
|
||||||
|
[JsonPropertyName("detail")] public string? Detail { get; set; }
|
||||||
|
[JsonPropertyName("sql")] public string? Sql { get; set; }
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>ResolveSpec envelope. <see cref="Data"/> is raw JSON; use <see cref="Decode{T}"/>.</summary>
|
||||||
|
public sealed class Response
|
||||||
|
{
|
||||||
|
[JsonPropertyName("success")] public bool Success { get; set; }
|
||||||
|
[JsonPropertyName("data")] public JsonElement Data { get; set; }
|
||||||
|
[JsonPropertyName("metadata")] public Metadata? Metadata { get; set; }
|
||||||
|
[JsonPropertyName("error")] public ApiError? Error { get; set; }
|
||||||
|
|
||||||
|
public T? Decode<T>() => Data.ValueKind == JsonValueKind.Undefined ? default : Data.Deserialize<T>();
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>Thrown on a non-2xx response or an unsuccessful API result.</summary>
|
||||||
|
public sealed class ResolveSpecException : Exception
|
||||||
|
{
|
||||||
|
public int StatusCode { get; }
|
||||||
|
public ApiError Error { get; }
|
||||||
|
|
||||||
|
public ResolveSpecException(string message, int statusCode, ApiError? error = null) : base(message)
|
||||||
|
{
|
||||||
|
StatusCode = statusCode;
|
||||||
|
Error = error ?? new ApiError { Message = message };
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,179 @@
|
|||||||
|
using System.Net;
|
||||||
|
using System.Text;
|
||||||
|
using System.Text.Json;
|
||||||
|
using ResolveSpec;
|
||||||
|
using Xunit;
|
||||||
|
|
||||||
|
public class Stub : HttpMessageHandler
|
||||||
|
{
|
||||||
|
public HttpRequestMessage? Request;
|
||||||
|
public string Body = "";
|
||||||
|
readonly HttpStatusCode _status;
|
||||||
|
readonly string _json;
|
||||||
|
readonly Dictionary<string, string> _headers;
|
||||||
|
|
||||||
|
public Stub(HttpStatusCode status, string json, Dictionary<string, string>? headers = null)
|
||||||
|
{
|
||||||
|
_status = status; _json = json; _headers = headers ?? new();
|
||||||
|
}
|
||||||
|
|
||||||
|
protected override async Task<HttpResponseMessage> SendAsync(HttpRequestMessage request, CancellationToken ct)
|
||||||
|
{
|
||||||
|
Request = request;
|
||||||
|
Body = request.Content == null ? "" : await request.Content.ReadAsStringAsync(ct);
|
||||||
|
var r = new HttpResponseMessage(_status) { Content = new StringContent(_json, Encoding.UTF8, "application/json") };
|
||||||
|
foreach (var (k, v) in _headers)
|
||||||
|
if (!r.Headers.TryAddWithoutValidation(k, v)) r.Content.Headers.TryAddWithoutValidation(k, v);
|
||||||
|
return r;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public class ResolveSpecTests
|
||||||
|
{
|
||||||
|
static (ResolveSpecClient, Stub) Make(HttpStatusCode s, string json)
|
||||||
|
{
|
||||||
|
var stub = new Stub(s, json);
|
||||||
|
var o = new ClientOptions { Token = "tok", HttpClient = new HttpClient(stub) };
|
||||||
|
o.Headers["X-Tenant"] = "a";
|
||||||
|
return (new ResolveSpecClient("http://localhost:3000/", o), stub);
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public async Task ReadPostsBody()
|
||||||
|
{
|
||||||
|
var (c, s) = Make(HttpStatusCode.OK, """{"success":true,"data":[{"id":1}]}""");
|
||||||
|
var r = await c.ReadAsync("public", "users", null, new Options { Limit = 5, Filters = new() { new FilterOption { Column = "a", Operator = "eq", Value = 1 } } });
|
||||||
|
Assert.Equal(HttpMethod.Post, s.Request!.Method);
|
||||||
|
Assert.Equal("/public/users", s.Request.RequestUri!.AbsolutePath);
|
||||||
|
Assert.Equal("Bearer tok", s.Request.Headers.Authorization!.ToString());
|
||||||
|
Assert.Equal("a", s.Request.Headers.GetValues("X-Tenant").Single());
|
||||||
|
using var body = JsonDocument.Parse(s.Body);
|
||||||
|
Assert.Equal("read", body.RootElement.GetProperty("operation").GetString());
|
||||||
|
Assert.Equal(5, body.RootElement.GetProperty("options").GetProperty("limit").GetInt32());
|
||||||
|
Assert.False(body.RootElement.TryGetProperty("id", out _));
|
||||||
|
Assert.Single(r.Decode<List<Dictionary<string, int>>>()!);
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public async Task IdPlacement()
|
||||||
|
{
|
||||||
|
var (c, s) = Make(HttpStatusCode.OK, """{"success":true,"data":{}}""");
|
||||||
|
await c.ReadAsync("s", "e", 7);
|
||||||
|
Assert.Equal("/s/e/7", s.Request!.RequestUri!.AbsolutePath);
|
||||||
|
await c.UpdateAsync("s", "e", new { a = 1 }, new[] { "1", "2" });
|
||||||
|
Assert.Equal("/s/e", s.Request!.RequestUri!.AbsolutePath);
|
||||||
|
using (var b = JsonDocument.Parse(s.Body))
|
||||||
|
{
|
||||||
|
Assert.Equal(2, b.RootElement.GetProperty("id").GetArrayLength());
|
||||||
|
Assert.Equal("update", b.RootElement.GetProperty("operation").GetString());
|
||||||
|
}
|
||||||
|
await c.DeleteAsync("s", "e", "a/b");
|
||||||
|
Assert.Equal("/s/e/a%2Fb", s.Request!.RequestUri!.AbsoluteUri[(s.Request.RequestUri.AbsoluteUri.IndexOf("/s/e", StringComparison.Ordinal))..]);
|
||||||
|
Assert.Contains("\"delete\"", s.Body);
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public async Task Errors()
|
||||||
|
{
|
||||||
|
var (c, _) = Make(HttpStatusCode.BadRequest, """{"success":false,"error":{"code":"x","message":"bad","detail":"why"}}""");
|
||||||
|
var e = await Assert.ThrowsAsync<ResolveSpecException>(() => c.ReadAsync("s", "e"));
|
||||||
|
Assert.Equal((400, "x", "bad", "why"), (e.StatusCode, e.Error.Code, e.Message, e.Error.Detail));
|
||||||
|
|
||||||
|
var (c2, _) = Make(HttpStatusCode.BadGateway, "bad gateway");
|
||||||
|
var e2 = await Assert.ThrowsAsync<ResolveSpecException>(() => c2.ReadAsync("s", "e"));
|
||||||
|
Assert.Equal((502, "bad gateway"), (e2.StatusCode, e2.Message));
|
||||||
|
|
||||||
|
var (c3, _) = Make(HttpStatusCode.OK, """{"success":false,"error":{"code":"c","message":"nope"}}""");
|
||||||
|
var e3 = await Assert.ThrowsAsync<ResolveSpecException>(() => c3.ReadAsync("s", "e"));
|
||||||
|
Assert.Equal("nope", e3.Message);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public class FuncSpecTests
|
||||||
|
{
|
||||||
|
[Fact]
|
||||||
|
public void HeaderFilters()
|
||||||
|
{
|
||||||
|
var h = FuncSpecClient.BuildHeaders(new FuncSpecOptions
|
||||||
|
{
|
||||||
|
Filters = new()
|
||||||
|
{
|
||||||
|
new() { Column = "status", Operator = "eq", Value = "active" },
|
||||||
|
new() { Column = "age", Operator = "gte", Value = 18 },
|
||||||
|
new() { Column = "name", Operator = "contains", Value = "x", LogicOperator = "OR" },
|
||||||
|
new() { Column = "deleted", Operator = "is_null" },
|
||||||
|
new() { Column = "id", Operator = "in", Value = new[] { 1, 2 } },
|
||||||
|
new() { Column = "p", Operator = "between_inclusive", Value = new[] { 1, 5 } },
|
||||||
|
},
|
||||||
|
});
|
||||||
|
Assert.Equal(new Dictionary<string, string>
|
||||||
|
{
|
||||||
|
["X-FieldFilter-status"] = "active",
|
||||||
|
["X-SearchOp-greaterthanorequal-age"] = "18",
|
||||||
|
["X-SearchOr-contains-name"] = "x",
|
||||||
|
["X-SearchOp-empty-deleted"] = "",
|
||||||
|
["X-SearchOp-in-id"] = "1,2",
|
||||||
|
["X-SearchOp-betweeninclusive-p"] = "1,5",
|
||||||
|
}, h);
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public void HeaderMiscAndEncoding()
|
||||||
|
{
|
||||||
|
var h = FuncSpecClient.BuildHeaders(new FuncSpecOptions
|
||||||
|
{
|
||||||
|
SearchFilters = new() { ["name"] = "bob" }, CustomSqlWhere = "a = 1", CustomSqlOr = "b = 2",
|
||||||
|
Sort = new() { new() { Column = "name", Direction = "asc" }, new() { Column = "created_at", Direction = "DESC" } },
|
||||||
|
Limit = 5, Offset = 10, Distinct = true, SkipCount = true, SkipCache = false, ResponseFormat = "syncfusion",
|
||||||
|
});
|
||||||
|
Assert.Equal("name ASC,created_at DESC", h["X-Sort"]);
|
||||||
|
Assert.Equal("bob", h["X-SearchFilter-name"]);
|
||||||
|
Assert.Equal("a = 1", h["X-Custom-SQL-W"]);
|
||||||
|
Assert.Equal("false", h["X-SkipCache"]);
|
||||||
|
Assert.Equal("true", h["X-Syncfusion"]);
|
||||||
|
|
||||||
|
h = FuncSpecClient.BuildHeaders(new FuncSpecOptions { Filters = new()
|
||||||
|
{
|
||||||
|
new() { Column = "n", Operator = "eq", Value = "héllo" },
|
||||||
|
new() { Column = "m", Operator = "eq", Value = " pad" },
|
||||||
|
} });
|
||||||
|
Assert.StartsWith("ZIP_", h["X-FieldFilter-n"]);
|
||||||
|
Assert.Equal("héllo", FuncSpecClient.DecodeHeaderValue(h["X-FieldFilter-n"]));
|
||||||
|
Assert.Equal(" pad", FuncSpecClient.DecodeHeaderValue(h["X-FieldFilter-m"]));
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public void QueryBuilding()
|
||||||
|
{
|
||||||
|
var q = FuncSpecClient.BuildQuery(new Dictionary<string, object?> { ["a"] = true, ["b"] = new[] { "x", "y" }, ["c"] = null, ["d"] = 3 });
|
||||||
|
Assert.Equal(new[] { "a=true", "b=x", "b=y", "d=3" }, q.Select(kv => $"{kv.Key}={kv.Value}"));
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public async Task QueryListMetadata()
|
||||||
|
{
|
||||||
|
var stub = new Stub((HttpStatusCode)206, """[{"id":1},{"id":2}]""", new() { ["Content-Range"] = "items 10-12/50" });
|
||||||
|
var c = new FuncSpecClient("http://x", new ClientOptions { Token = "tok", HttpClient = new HttpClient(stub) });
|
||||||
|
var r = await c.QueryListAsync("/api/users", new Dictionary<string, object?> { ["org"] = 1 }, new FuncSpecOptions { Limit = 2 });
|
||||||
|
Assert.Equal("GET", stub.Request!.Method.Method);
|
||||||
|
Assert.Equal("/api/users", stub.Request.RequestUri!.AbsolutePath);
|
||||||
|
Assert.Equal("?org=1", stub.Request.RequestUri.Query);
|
||||||
|
Assert.Equal("2", stub.Request.Headers.GetValues("X-Limit").Single());
|
||||||
|
Assert.Equal((50L, 2L, 50L, 2L, 10L), (r.Metadata!.Total, r.Metadata.Count, r.Metadata.Filtered, r.Metadata.Limit, r.Metadata.Offset));
|
||||||
|
Assert.Equal(2, r.Data.GetArrayLength());
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public async Task QuerySingleAndError()
|
||||||
|
{
|
||||||
|
var ok = new FuncSpecClient("http://x", new ClientOptions { HttpClient = new HttpClient(new Stub(HttpStatusCode.OK, """{"id":1}""")) });
|
||||||
|
var r = await ok.QueryAsync("api/u");
|
||||||
|
Assert.Null(r.Metadata);
|
||||||
|
Assert.Equal(1, r.Data.GetProperty("id").GetInt32());
|
||||||
|
|
||||||
|
var bad = new FuncSpecClient("http://x", new ClientOptions { HttpClient = new HttpClient(new Stub(HttpStatusCode.BadRequest,
|
||||||
|
"""{"success":false,"error":{"code":"hook_error","message":"Hook execution failed","detail":"authentication required"}}""")) });
|
||||||
|
var e = await Assert.ThrowsAsync<ResolveSpecException>(() => bad.QueryAsync("api/u"));
|
||||||
|
Assert.Equal(("hook_error", "authentication required"), (e.Error.Code, e.Error.Detail));
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,16 @@
|
|||||||
|
<Project Sdk="Microsoft.NET.Sdk">
|
||||||
|
<PropertyGroup>
|
||||||
|
<TargetFramework>net8.0</TargetFramework>
|
||||||
|
<Nullable>enable</Nullable>
|
||||||
|
<ImplicitUsings>enable</ImplicitUsings>
|
||||||
|
<IsPackable>false</IsPackable>
|
||||||
|
</PropertyGroup>
|
||||||
|
<ItemGroup>
|
||||||
|
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.11.1" />
|
||||||
|
<PackageReference Include="xunit" Version="2.9.2" />
|
||||||
|
<PackageReference Include="xunit.runner.visualstudio" Version="2.8.2" />
|
||||||
|
</ItemGroup>
|
||||||
|
<ItemGroup>
|
||||||
|
<ProjectReference Include="../src/ResolveSpec.csproj" />
|
||||||
|
</ItemGroup>
|
||||||
|
</Project>
|
||||||
@@ -0,0 +1,3 @@
|
|||||||
|
.dart_tool/
|
||||||
|
pubspec.lock
|
||||||
|
build/
|
||||||
@@ -0,0 +1,41 @@
|
|||||||
|
# resolvespec (Dart)
|
||||||
|
|
||||||
|
Dart / Flutter client for ResolveSpec (JSON body) and FunctionSpec. Depends on `package:http`. Dart >= 3.3.
|
||||||
|
|
||||||
|
## Clients
|
||||||
|
|
||||||
|
| Type | Constructor | Methods |
|
||||||
|
|---|---|---|
|
||||||
|
| `ResolveSpecClient` | `(baseUrl, [ClientOptions])` | `getMetadata` `read` `create` `update` `delete` `close` |
|
||||||
|
| `FuncSpecClient` | `(baseUrl, [ClientOptions])` | `query` `queryList` `close` |
|
||||||
|
|
||||||
|
`ClientOptions(token:, headers:, timeout:, httpClient:)`. Precedence: Content-Type < custom headers < bearer token.
|
||||||
|
|
||||||
|
## ResolveSpec
|
||||||
|
|
||||||
|
- `id`: `int`/`String` → URL, `List<String>` → body. Named args: `id:`, `options:`.
|
||||||
|
- `Options`, `FilterOption(column, operator, [value, logic])`, `SortOption(column, [direction])`.
|
||||||
|
- Result: `Response{success, data (decoded JSON), metadata}`.
|
||||||
|
|
||||||
|
## FunctionSpec
|
||||||
|
|
||||||
|
- Routes are server-defined: pass the `path`.
|
||||||
|
- `params:` map → query string (list → repeated keys, null skipped).
|
||||||
|
- `FuncSpecOptions` → `X-*` headers: `filters`, `searchFilters`, `customSqlWhere`, `customSqlOr`, `sort`, `limit`, `offset`, `distinct`, `skipCount`, `skipCache`, `responseFormat`.
|
||||||
|
- `queryList` fills `metadata` from `Content-Range`; 206 is success.
|
||||||
|
- Helpers: `buildHeaders`, `buildQuery`, `encodeHeaderValue`, `decodeHeaderValue`.
|
||||||
|
|
||||||
|
## Server quirks
|
||||||
|
|
||||||
|
- `sort` is raw SQL in ORDER BY (client sends `col ASC|DESC`).
|
||||||
|
- One search operator per column.
|
||||||
|
- Values starting `ZIP_` / `__` are base64-decoded by the server.
|
||||||
|
- Non-ASCII, control chars and edge spaces are auto-encoded (`ZIP_`).
|
||||||
|
|
||||||
|
## Errors
|
||||||
|
|
||||||
|
`ResolveSpecException{statusCode, message, error: ApiError{code, detail, sql}}`.
|
||||||
|
|
||||||
|
## Test
|
||||||
|
|
||||||
|
`dart test`
|
||||||
@@ -0,0 +1 @@
|
|||||||
|
include: package:lints/recommended.yaml
|
||||||
@@ -0,0 +1,7 @@
|
|||||||
|
/// Client for ResolveSpec (JSON body) and FunctionSpec endpoints.
|
||||||
|
library;
|
||||||
|
|
||||||
|
export 'src/client.dart' show ClientOptions, ResolveSpecException;
|
||||||
|
export 'src/funcspec.dart';
|
||||||
|
export 'src/resolvespec.dart';
|
||||||
|
export 'src/types.dart';
|
||||||
@@ -0,0 +1,98 @@
|
|||||||
|
import 'dart:convert';
|
||||||
|
|
||||||
|
import 'package:http/http.dart' as http;
|
||||||
|
|
||||||
|
import 'types.dart';
|
||||||
|
|
||||||
|
/// Thrown on a non-2xx response or an unsuccessful API result.
|
||||||
|
class ResolveSpecException implements Exception {
|
||||||
|
final int statusCode;
|
||||||
|
final String message;
|
||||||
|
final ApiError error;
|
||||||
|
|
||||||
|
ResolveSpecException(this.message, this.statusCode, [ApiError? error])
|
||||||
|
: error = error ?? ApiError(message: message);
|
||||||
|
|
||||||
|
@override
|
||||||
|
String toString() => 'ResolveSpecException($statusCode): $message';
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Shared HTTP configuration for both clients.
|
||||||
|
class ClientOptions {
|
||||||
|
final String? token;
|
||||||
|
final Map<String, String> headers;
|
||||||
|
final Duration timeout;
|
||||||
|
|
||||||
|
/// Supply your own client (tests, pooling).
|
||||||
|
final http.Client? httpClient;
|
||||||
|
|
||||||
|
const ClientOptions(
|
||||||
|
{this.token,
|
||||||
|
this.headers = const {},
|
||||||
|
this.timeout = const Duration(seconds: 30),
|
||||||
|
this.httpClient});
|
||||||
|
}
|
||||||
|
|
||||||
|
class Transport {
|
||||||
|
final String baseUrl;
|
||||||
|
final ClientOptions options;
|
||||||
|
final http.Client _http;
|
||||||
|
|
||||||
|
Transport(String baseUrl, ClientOptions? options)
|
||||||
|
: baseUrl = baseUrl.replaceAll(RegExp(r'/+$'), ''),
|
||||||
|
options = options ?? const ClientOptions(),
|
||||||
|
_http = options?.httpClient ?? http.Client();
|
||||||
|
|
||||||
|
/// Content-Type < custom headers < per-call headers < bearer token.
|
||||||
|
Future<http.Response> send(String method, Uri uri,
|
||||||
|
{String? body, Map<String, String>? extra}) {
|
||||||
|
final headers = <String, String>{'Content-Type': 'application/json'};
|
||||||
|
void merge(Map<String, String> src) {
|
||||||
|
for (final e in src.entries) {
|
||||||
|
headers.removeWhere((k, _) => k.toLowerCase() == e.key.toLowerCase());
|
||||||
|
headers[e.key] = e.value;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
merge(options.headers);
|
||||||
|
if (extra != null) merge(extra);
|
||||||
|
final token = options.token;
|
||||||
|
if (token != null && token.isNotEmpty) {
|
||||||
|
merge({'Authorization': 'Bearer $token'});
|
||||||
|
}
|
||||||
|
|
||||||
|
final req = http.Request(method, uri)..headers.addAll(headers);
|
||||||
|
if (body != null) req.body = body;
|
||||||
|
return _http
|
||||||
|
.send(req)
|
||||||
|
.timeout(options.timeout)
|
||||||
|
.then(http.Response.fromStream);
|
||||||
|
}
|
||||||
|
|
||||||
|
void close() => _http.close();
|
||||||
|
|
||||||
|
static ResolveSpecException errorFrom(http.Response resp) {
|
||||||
|
final body = utf8.decode(resp.bodyBytes, allowMalformed: true);
|
||||||
|
ApiError? err;
|
||||||
|
var isJson = false;
|
||||||
|
try {
|
||||||
|
final parsed = jsonDecode(body);
|
||||||
|
isJson = true;
|
||||||
|
if (parsed is Map<String, dynamic> &&
|
||||||
|
parsed['error'] is Map<String, dynamic>) {
|
||||||
|
err = ApiError.fromJson(parsed['error'] as Map<String, dynamic>);
|
||||||
|
}
|
||||||
|
} on FormatException {
|
||||||
|
// not JSON
|
||||||
|
}
|
||||||
|
var message = err?.message ?? '';
|
||||||
|
if (message.isEmpty) {
|
||||||
|
var text = isJson ? '' : body.trim();
|
||||||
|
if (text.length > 200) text = text.substring(0, 200);
|
||||||
|
message = text.isNotEmpty
|
||||||
|
? text
|
||||||
|
: '${resp.reasonPhrase ?? 'Error'} (${resp.statusCode})';
|
||||||
|
}
|
||||||
|
return ResolveSpecException(message, resp.statusCode, err);
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,223 @@
|
|||||||
|
import 'dart:convert';
|
||||||
|
|
||||||
|
import 'client.dart';
|
||||||
|
import 'types.dart';
|
||||||
|
|
||||||
|
/// Options sent to funcspec endpoints as X-* headers.
|
||||||
|
///
|
||||||
|
/// Server behaviour (pkg/funcspec): [sort] is inserted raw into ORDER BY (so it is sent as SQL
|
||||||
|
/// terms); only one search operator per column is kept; values starting with `ZIP_` or `__`
|
||||||
|
/// are base64-decoded by the server, so such plaintext values cannot be sent faithfully.
|
||||||
|
class FuncSpecOptions {
|
||||||
|
/// eq+AND -> X-FieldFilter; others X-SearchOp / X-SearchOr.
|
||||||
|
final List<FilterOption>? filters;
|
||||||
|
|
||||||
|
/// X-SearchFilter-{col}: text ILIKE.
|
||||||
|
final Map<String, String>? searchFilters;
|
||||||
|
final String? customSqlWhere;
|
||||||
|
final String? customSqlOr;
|
||||||
|
final List<SortOption>? sort;
|
||||||
|
final int? limit;
|
||||||
|
final int? offset;
|
||||||
|
final bool? distinct;
|
||||||
|
final bool? skipCount;
|
||||||
|
final bool? skipCache;
|
||||||
|
|
||||||
|
/// simple | detail | syncfusion
|
||||||
|
final String? responseFormat;
|
||||||
|
|
||||||
|
const FuncSpecOptions({
|
||||||
|
this.filters,
|
||||||
|
this.searchFilters,
|
||||||
|
this.customSqlWhere,
|
||||||
|
this.customSqlOr,
|
||||||
|
this.sort,
|
||||||
|
this.limit,
|
||||||
|
this.offset,
|
||||||
|
this.distinct,
|
||||||
|
this.skipCount,
|
||||||
|
this.skipCache,
|
||||||
|
this.responseFormat,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
const _operatorMap = {
|
||||||
|
'eq': 'equals',
|
||||||
|
'neq': 'notequals',
|
||||||
|
'gt': 'greaterthan',
|
||||||
|
'gte': 'greaterthanorequal',
|
||||||
|
'lt': 'lessthan',
|
||||||
|
'lte': 'lessthanorequal',
|
||||||
|
'like': 'contains',
|
||||||
|
'ilike': 'contains',
|
||||||
|
'contains': 'contains',
|
||||||
|
'startswith': 'beginswith',
|
||||||
|
'endswith': 'endswith',
|
||||||
|
'in': 'in',
|
||||||
|
'between': 'between',
|
||||||
|
'between_inclusive': 'betweeninclusive',
|
||||||
|
'is_null': 'empty',
|
||||||
|
'is_not_null': 'notempty',
|
||||||
|
};
|
||||||
|
|
||||||
|
String _scalar(Object? v) => v == null ? '' : v.toString();
|
||||||
|
|
||||||
|
String _filterValue(Object? v) =>
|
||||||
|
v is Iterable ? v.map(_scalar).join(',') : _scalar(v);
|
||||||
|
|
||||||
|
/// Base64 (UTF-8) with the `ZIP_` prefix.
|
||||||
|
String encodeHeaderValue(String v) => 'ZIP_${base64.encode(utf8.encode(v))}';
|
||||||
|
|
||||||
|
/// Decode a value that may carry a `ZIP_` or `__` prefix (nested allowed).
|
||||||
|
String decodeHeaderValue(String v) {
|
||||||
|
for (final p in const ['ZIP_', '__']) {
|
||||||
|
if (v.startsWith(p)) {
|
||||||
|
var b64 = v.substring(p.length).replaceAll(RegExp(r'[\n\r ]'), '');
|
||||||
|
b64 = b64.padRight(b64.length + (4 - b64.length % 4) % 4, '=');
|
||||||
|
try {
|
||||||
|
return decodeHeaderValue(utf8.decode(base64.decode(b64)));
|
||||||
|
} on FormatException {
|
||||||
|
return v;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return v;
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Encode values that are unsafe as raw header/query text (non-ASCII, control chars, edge spaces).
|
||||||
|
String _safe(String v) {
|
||||||
|
final unsafe =
|
||||||
|
v != v.trim() || v.runes.any((c) => c > 127 || c < 32 || c == 127);
|
||||||
|
return unsafe ? encodeHeaderValue(v) : v;
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Build the X-* headers understood by funcspec.ParseParameters.
|
||||||
|
Map<String, String> buildHeaders(FuncSpecOptions? o) {
|
||||||
|
final h = <String, String>{};
|
||||||
|
if (o == null) return h;
|
||||||
|
|
||||||
|
for (final f in o.filters ?? const <FilterOption>[]) {
|
||||||
|
final logic = f.logicOperator ?? 'AND';
|
||||||
|
final v = _safe(_filterValue(f.value));
|
||||||
|
if (f.operator == 'eq' && logic == 'AND') {
|
||||||
|
h['X-FieldFilter-${f.column}'] = v;
|
||||||
|
} else {
|
||||||
|
final kind = logic == 'OR' ? 'X-SearchOr' : 'X-SearchOp';
|
||||||
|
h['$kind-${_operatorMap[f.operator] ?? f.operator}-${f.column}'] = v;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
o.searchFilters
|
||||||
|
?.forEach((col, text) => h['X-SearchFilter-$col'] = _safe(text));
|
||||||
|
if (o.customSqlWhere != null && o.customSqlWhere!.isNotEmpty) {
|
||||||
|
h['X-Custom-SQL-W'] = _safe(o.customSqlWhere!);
|
||||||
|
}
|
||||||
|
if (o.customSqlOr != null && o.customSqlOr!.isNotEmpty) {
|
||||||
|
h['X-Custom-SQL-Or'] = _safe(o.customSqlOr!);
|
||||||
|
}
|
||||||
|
if (o.sort != null && o.sort!.isNotEmpty) {
|
||||||
|
// funcspec puts this verbatim into ORDER BY
|
||||||
|
h['X-Sort'] = _safe(o.sort!
|
||||||
|
.map((s) =>
|
||||||
|
'${s.column} ${s.direction.toLowerCase() == 'desc' ? 'DESC' : 'ASC'}')
|
||||||
|
.join(','));
|
||||||
|
}
|
||||||
|
if (o.limit != null) h['X-Limit'] = '${o.limit}';
|
||||||
|
if (o.offset != null) h['X-Offset'] = '${o.offset}';
|
||||||
|
if (o.distinct != null) h['X-Distinct'] = '${o.distinct}';
|
||||||
|
if (o.skipCount != null) h['X-SkipCount'] = '${o.skipCount}';
|
||||||
|
if (o.skipCache != null) h['X-SkipCache'] = '${o.skipCache}';
|
||||||
|
switch (o.responseFormat) {
|
||||||
|
case 'simple':
|
||||||
|
h['X-SimpleApi'] = 'true';
|
||||||
|
case 'detail':
|
||||||
|
h['X-DetailApi'] = 'true';
|
||||||
|
case 'syncfusion':
|
||||||
|
h['X-Syncfusion'] = 'true';
|
||||||
|
}
|
||||||
|
return h;
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Build query-string pairs: lists -> repeated keys, null skipped, bools -> true/false.
|
||||||
|
Map<String, List<String>> buildQuery(Map<String, Object?>? params) {
|
||||||
|
final out = <String, List<String>>{};
|
||||||
|
params?.forEach((k, v) {
|
||||||
|
if (v == null) return;
|
||||||
|
out[k] = v is Iterable
|
||||||
|
? v.map((e) => _safe(_scalar(e))).toList()
|
||||||
|
: [_safe(_scalar(v))];
|
||||||
|
});
|
||||||
|
return out;
|
||||||
|
}
|
||||||
|
|
||||||
|
String? _header(Map<String, String> headers, String name) {
|
||||||
|
for (final e in headers.entries) {
|
||||||
|
if (e.key.toLowerCase() == name) return e.value;
|
||||||
|
}
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
final _contentRange = RegExp(r'(\d+)-(\d+)/(\d+)');
|
||||||
|
|
||||||
|
Metadata _metadata(String? contentRange, FuncSpecOptions? o) {
|
||||||
|
final m = _contentRange.firstMatch(contentRange ?? '');
|
||||||
|
if (m == null) return Metadata(limit: o?.limit ?? 0);
|
||||||
|
final start = int.parse(m.group(1)!);
|
||||||
|
final end = int.parse(m.group(2)!);
|
||||||
|
final total = int.parse(m.group(3)!);
|
||||||
|
return Metadata(
|
||||||
|
total: total,
|
||||||
|
count: end - start,
|
||||||
|
filtered: total,
|
||||||
|
limit: o?.limit ?? 0,
|
||||||
|
offset: start);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Client for user-defined SQL endpoints. Routes are defined by the server application.
|
||||||
|
class FuncSpecClient {
|
||||||
|
final Transport _t;
|
||||||
|
|
||||||
|
FuncSpecClient(String baseUrl, [ClientOptions? options])
|
||||||
|
: _t = Transport(baseUrl, options);
|
||||||
|
|
||||||
|
void close() => _t.close();
|
||||||
|
|
||||||
|
Future<Response> _call(String method, String path,
|
||||||
|
Map<String, Object?>? params, FuncSpecOptions? o, bool list) async {
|
||||||
|
final base =
|
||||||
|
Uri.parse('${_t.baseUrl}/${path.replaceAll(RegExp(r'^/+'), '')}');
|
||||||
|
final pairs = <String>[];
|
||||||
|
buildQuery(params).forEach((k, vs) {
|
||||||
|
for (final v in vs) {
|
||||||
|
pairs.add(
|
||||||
|
'${Uri.encodeQueryComponent(k)}=${Uri.encodeQueryComponent(v)}');
|
||||||
|
}
|
||||||
|
});
|
||||||
|
final uri = pairs.isEmpty ? base : base.replace(query: pairs.join('&'));
|
||||||
|
|
||||||
|
final resp = await _t.send(method, uri, extra: buildHeaders(o));
|
||||||
|
if (resp.statusCode < 200 || resp.statusCode > 299) {
|
||||||
|
throw Transport.errorFrom(resp); // 206 is success
|
||||||
|
}
|
||||||
|
final text = utf8.decode(resp.bodyBytes);
|
||||||
|
return Response(
|
||||||
|
success: true,
|
||||||
|
data: text.trim().isEmpty ? null : jsonDecode(text),
|
||||||
|
metadata:
|
||||||
|
list ? _metadata(_header(resp.headers, 'content-range'), o) : null,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Single-record endpoint (SqlQuery). `data` is the row object.
|
||||||
|
Future<Response> query(String path,
|
||||||
|
{Map<String, Object?>? params,
|
||||||
|
FuncSpecOptions? options,
|
||||||
|
String method = 'GET'}) =>
|
||||||
|
_call(method.toUpperCase(), path, params, options, false);
|
||||||
|
|
||||||
|
/// List endpoint (SqlQueryList). Metadata comes from Content-Range.
|
||||||
|
Future<Response> queryList(String path,
|
||||||
|
{Map<String, Object?>? params,
|
||||||
|
FuncSpecOptions? options,
|
||||||
|
String method = 'GET'}) =>
|
||||||
|
_call(method.toUpperCase(), path, params, options, true);
|
||||||
|
}
|
||||||
@@ -0,0 +1,88 @@
|
|||||||
|
import 'dart:convert';
|
||||||
|
|
||||||
|
import 'client.dart';
|
||||||
|
import 'types.dart';
|
||||||
|
|
||||||
|
/// Client for the ResolveSpec JSON body protocol: POST {operation, data, options}.
|
||||||
|
///
|
||||||
|
/// A record `id` of type `int` or `String` goes in the URL; a `List<String>` goes in the body.
|
||||||
|
class ResolveSpecClient {
|
||||||
|
final Transport _t;
|
||||||
|
|
||||||
|
ResolveSpecClient(String baseUrl, [ClientOptions? options])
|
||||||
|
: _t = Transport(baseUrl, options);
|
||||||
|
|
||||||
|
void close() => _t.close();
|
||||||
|
|
||||||
|
static String? _urlId(Object? id) =>
|
||||||
|
id == null || id is List ? null : id.toString();
|
||||||
|
|
||||||
|
static List<String>? _bodyId(Object? id) =>
|
||||||
|
id is List ? id.map((e) => e.toString()).toList() : null;
|
||||||
|
|
||||||
|
Uri _url(String schema, String entity, String? id) {
|
||||||
|
var u =
|
||||||
|
'${_t.baseUrl}/${Uri.encodeComponent(schema)}/${Uri.encodeComponent(entity)}';
|
||||||
|
if (id != null && id.isNotEmpty) u += '/${Uri.encodeComponent(id)}';
|
||||||
|
return Uri.parse(u);
|
||||||
|
}
|
||||||
|
|
||||||
|
Future<Response> _send(
|
||||||
|
String method, Uri url, Map<String, dynamic>? body) async {
|
||||||
|
final resp = await _t.send(method, url,
|
||||||
|
body: body == null ? null : jsonEncode(body));
|
||||||
|
if (resp.statusCode < 200 || resp.statusCode > 299) {
|
||||||
|
throw Transport.errorFrom(resp);
|
||||||
|
}
|
||||||
|
final decoded = jsonDecode(utf8.decode(resp.bodyBytes));
|
||||||
|
final r = Response.fromJson(decoded as Map<String, dynamic>);
|
||||||
|
if (!r.success && r.error != null) {
|
||||||
|
throw ResolveSpecException(r.error!.message, resp.statusCode, r.error);
|
||||||
|
}
|
||||||
|
return r;
|
||||||
|
}
|
||||||
|
|
||||||
|
/// GET /{schema}/{entity}
|
||||||
|
Future<Response> getMetadata(String schema, String entity) =>
|
||||||
|
_send('GET', _url(schema, entity, null), null);
|
||||||
|
|
||||||
|
Future<Response> read(String schema, String entity,
|
||||||
|
{Object? id, Options? options}) =>
|
||||||
|
_send(
|
||||||
|
'POST',
|
||||||
|
_url(schema, entity, _urlId(id)),
|
||||||
|
{
|
||||||
|
'operation': 'read',
|
||||||
|
if (_bodyId(id) != null) 'id': _bodyId(id),
|
||||||
|
if (options != null) 'options': options.toJson()
|
||||||
|
},
|
||||||
|
);
|
||||||
|
|
||||||
|
Future<Response> create(String schema, String entity, Object data,
|
||||||
|
{Options? options}) =>
|
||||||
|
_send(
|
||||||
|
'POST',
|
||||||
|
_url(schema, entity, null),
|
||||||
|
{
|
||||||
|
'operation': 'create',
|
||||||
|
'data': data,
|
||||||
|
if (options != null) 'options': options.toJson()
|
||||||
|
},
|
||||||
|
);
|
||||||
|
|
||||||
|
Future<Response> update(String schema, String entity, Object data,
|
||||||
|
{Object? id, Options? options}) =>
|
||||||
|
_send(
|
||||||
|
'POST',
|
||||||
|
_url(schema, entity, _urlId(id)),
|
||||||
|
{
|
||||||
|
'operation': 'update',
|
||||||
|
if (_bodyId(id) != null) 'id': _bodyId(id),
|
||||||
|
'data': data,
|
||||||
|
if (options != null) 'options': options.toJson(),
|
||||||
|
},
|
||||||
|
);
|
||||||
|
|
||||||
|
Future<Response> delete(String schema, String entity, Object id) =>
|
||||||
|
_send('POST', _url(schema, entity, _urlId(id)), {'operation': 'delete'});
|
||||||
|
}
|
||||||
@@ -0,0 +1,287 @@
|
|||||||
|
// Types aligned with Go pkg/common/types.go. toJson() emits the wire names.
|
||||||
|
|
||||||
|
Map<String, dynamic> _compact(Map<String, dynamic> m) {
|
||||||
|
m.removeWhere((_, v) => v == null);
|
||||||
|
return m;
|
||||||
|
}
|
||||||
|
|
||||||
|
class FilterOption {
|
||||||
|
final String column;
|
||||||
|
|
||||||
|
/// eq neq gt gte lt lte like ilike in contains startswith endswith between
|
||||||
|
/// between_inclusive is_null is_not_null
|
||||||
|
final String operator;
|
||||||
|
final Object? value;
|
||||||
|
|
||||||
|
/// AND | OR
|
||||||
|
final String? logicOperator;
|
||||||
|
|
||||||
|
const FilterOption(this.column, this.operator,
|
||||||
|
[this.value, this.logicOperator]);
|
||||||
|
|
||||||
|
Map<String, dynamic> toJson() => _compact({
|
||||||
|
'column': column,
|
||||||
|
'operator': operator,
|
||||||
|
'value': value,
|
||||||
|
'logic_operator': logicOperator,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
class SortOption {
|
||||||
|
final String column;
|
||||||
|
|
||||||
|
/// asc | desc
|
||||||
|
final String direction;
|
||||||
|
|
||||||
|
const SortOption(this.column, [this.direction = 'asc']);
|
||||||
|
|
||||||
|
Map<String, dynamic> toJson() => {'column': column, 'direction': direction};
|
||||||
|
}
|
||||||
|
|
||||||
|
class Parameter {
|
||||||
|
final String name;
|
||||||
|
final String value;
|
||||||
|
final int? sequence;
|
||||||
|
|
||||||
|
const Parameter(this.name, this.value, [this.sequence]);
|
||||||
|
|
||||||
|
Map<String, dynamic> toJson() =>
|
||||||
|
_compact({'name': name, 'value': value, 'sequence': sequence});
|
||||||
|
}
|
||||||
|
|
||||||
|
class CustomOperator {
|
||||||
|
final String name;
|
||||||
|
final String sql;
|
||||||
|
|
||||||
|
const CustomOperator(this.name, this.sql);
|
||||||
|
|
||||||
|
Map<String, dynamic> toJson() => {'name': name, 'sql': sql};
|
||||||
|
}
|
||||||
|
|
||||||
|
class ComputedColumn {
|
||||||
|
final String name;
|
||||||
|
final String expression;
|
||||||
|
|
||||||
|
const ComputedColumn(this.name, this.expression);
|
||||||
|
|
||||||
|
Map<String, dynamic> toJson() => {'name': name, 'expression': expression};
|
||||||
|
}
|
||||||
|
|
||||||
|
class PreloadOption {
|
||||||
|
final String? relation;
|
||||||
|
final String? tableName;
|
||||||
|
final List<String>? columns;
|
||||||
|
final List<String>? omitColumns;
|
||||||
|
final List<SortOption>? sort;
|
||||||
|
final List<FilterOption>? filters;
|
||||||
|
final String? where;
|
||||||
|
final int? limit;
|
||||||
|
final int? offset;
|
||||||
|
final bool? updateable;
|
||||||
|
final Map<String, String>? computedQl;
|
||||||
|
final bool? recursive;
|
||||||
|
final String? primaryKey;
|
||||||
|
final String? relatedKey;
|
||||||
|
final String? foreignKey;
|
||||||
|
final String? recursiveChildKey;
|
||||||
|
final List<String>? sqlJoins;
|
||||||
|
final List<String>? joinAliases;
|
||||||
|
|
||||||
|
const PreloadOption({
|
||||||
|
this.relation,
|
||||||
|
this.tableName,
|
||||||
|
this.columns,
|
||||||
|
this.omitColumns,
|
||||||
|
this.sort,
|
||||||
|
this.filters,
|
||||||
|
this.where,
|
||||||
|
this.limit,
|
||||||
|
this.offset,
|
||||||
|
this.updateable,
|
||||||
|
this.computedQl,
|
||||||
|
this.recursive,
|
||||||
|
this.primaryKey,
|
||||||
|
this.relatedKey,
|
||||||
|
this.foreignKey,
|
||||||
|
this.recursiveChildKey,
|
||||||
|
this.sqlJoins,
|
||||||
|
this.joinAliases,
|
||||||
|
});
|
||||||
|
|
||||||
|
Map<String, dynamic> toJson() => _compact({
|
||||||
|
'relation': relation,
|
||||||
|
'table_name': tableName,
|
||||||
|
'columns': columns,
|
||||||
|
'omit_columns': omitColumns,
|
||||||
|
'sort': sort?.map((e) => e.toJson()).toList(),
|
||||||
|
'filters': filters?.map((e) => e.toJson()).toList(),
|
||||||
|
'where': where,
|
||||||
|
'limit': limit,
|
||||||
|
'offset': offset,
|
||||||
|
'updateable': updateable,
|
||||||
|
'computed_ql': computedQl,
|
||||||
|
'recursive': recursive,
|
||||||
|
'primary_key': primaryKey,
|
||||||
|
'related_key': relatedKey,
|
||||||
|
'foreign_key': foreignKey,
|
||||||
|
'recursive_child_key': recursiveChildKey,
|
||||||
|
'sql_joins': sqlJoins,
|
||||||
|
'join_aliases': joinAliases,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
class VectorSearchOption {
|
||||||
|
final String column;
|
||||||
|
final List<double> vector;
|
||||||
|
|
||||||
|
/// l2 (default) | cosine | ip
|
||||||
|
final String? metric;
|
||||||
|
|
||||||
|
/// Distance column alias, default _distance.
|
||||||
|
final String? as;
|
||||||
|
final String? direction;
|
||||||
|
|
||||||
|
const VectorSearchOption(this.column, this.vector,
|
||||||
|
{this.metric, this.as, this.direction});
|
||||||
|
|
||||||
|
Map<String, dynamic> toJson() => _compact({
|
||||||
|
'column': column,
|
||||||
|
'vector': vector,
|
||||||
|
'metric': metric,
|
||||||
|
'as': as,
|
||||||
|
'direction': direction
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
/// ResolveSpec request options object.
|
||||||
|
class Options {
|
||||||
|
final List<PreloadOption>? preload;
|
||||||
|
final List<String>? columns;
|
||||||
|
final List<String>? omitColumns;
|
||||||
|
final List<FilterOption>? filters;
|
||||||
|
final List<SortOption>? sort;
|
||||||
|
final int? limit;
|
||||||
|
final int? offset;
|
||||||
|
final List<CustomOperator>? customOperators;
|
||||||
|
final List<ComputedColumn>? computedColumns;
|
||||||
|
final List<Parameter>? parameters;
|
||||||
|
final String? cursorForward;
|
||||||
|
final String? cursorBackward;
|
||||||
|
final String? fetchRowNumber;
|
||||||
|
final VectorSearchOption? vectorSearch;
|
||||||
|
|
||||||
|
const Options({
|
||||||
|
this.preload,
|
||||||
|
this.columns,
|
||||||
|
this.omitColumns,
|
||||||
|
this.filters,
|
||||||
|
this.sort,
|
||||||
|
this.limit,
|
||||||
|
this.offset,
|
||||||
|
this.customOperators,
|
||||||
|
this.computedColumns,
|
||||||
|
this.parameters,
|
||||||
|
this.cursorForward,
|
||||||
|
this.cursorBackward,
|
||||||
|
this.fetchRowNumber,
|
||||||
|
this.vectorSearch,
|
||||||
|
});
|
||||||
|
|
||||||
|
Map<String, dynamic> toJson() => _compact({
|
||||||
|
'preload': preload?.map((e) => e.toJson()).toList(),
|
||||||
|
'columns': columns,
|
||||||
|
'omit_columns': omitColumns,
|
||||||
|
'filters': filters?.map((e) => e.toJson()).toList(),
|
||||||
|
'sort': sort?.map((e) => e.toJson()).toList(),
|
||||||
|
'limit': limit,
|
||||||
|
'offset': offset,
|
||||||
|
'customOperators': customOperators?.map((e) => e.toJson()).toList(),
|
||||||
|
'computedColumns': computedColumns?.map((e) => e.toJson()).toList(),
|
||||||
|
'parameters': parameters?.map((e) => e.toJson()).toList(),
|
||||||
|
'cursor_forward': cursorForward,
|
||||||
|
'cursor_backward': cursorBackward,
|
||||||
|
'fetch_row_number': fetchRowNumber,
|
||||||
|
'vector_search': vectorSearch?.toJson(),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
class Metadata {
|
||||||
|
final int total;
|
||||||
|
final int count;
|
||||||
|
final int filtered;
|
||||||
|
final int limit;
|
||||||
|
final int offset;
|
||||||
|
|
||||||
|
const Metadata(
|
||||||
|
{this.total = 0,
|
||||||
|
this.count = 0,
|
||||||
|
this.filtered = 0,
|
||||||
|
this.limit = 0,
|
||||||
|
this.offset = 0});
|
||||||
|
|
||||||
|
factory Metadata.fromJson(Map<String, dynamic> j) => Metadata(
|
||||||
|
total: (j['total'] as num?)?.toInt() ?? 0,
|
||||||
|
count: (j['count'] as num?)?.toInt() ?? 0,
|
||||||
|
filtered: (j['filtered'] as num?)?.toInt() ?? 0,
|
||||||
|
limit: (j['limit'] as num?)?.toInt() ?? 0,
|
||||||
|
offset: (j['offset'] as num?)?.toInt() ?? 0,
|
||||||
|
);
|
||||||
|
|
||||||
|
@override
|
||||||
|
bool operator ==(Object other) =>
|
||||||
|
other is Metadata &&
|
||||||
|
other.total == total &&
|
||||||
|
other.count == count &&
|
||||||
|
other.filtered == filtered &&
|
||||||
|
other.limit == limit &&
|
||||||
|
other.offset == offset;
|
||||||
|
|
||||||
|
@override
|
||||||
|
int get hashCode => Object.hash(total, count, filtered, limit, offset);
|
||||||
|
|
||||||
|
@override
|
||||||
|
String toString() =>
|
||||||
|
'Metadata(total: $total, count: $count, filtered: $filtered, limit: $limit, offset: $offset)';
|
||||||
|
}
|
||||||
|
|
||||||
|
class ApiError {
|
||||||
|
final String code;
|
||||||
|
final String message;
|
||||||
|
final Object? details;
|
||||||
|
|
||||||
|
/// Server-side reason (funcspec / restheadspec).
|
||||||
|
final String? detail;
|
||||||
|
final String? sql;
|
||||||
|
|
||||||
|
const ApiError(
|
||||||
|
{this.code = '', this.message = '', this.details, this.detail, this.sql});
|
||||||
|
|
||||||
|
factory ApiError.fromJson(Map<String, dynamic> j) => ApiError(
|
||||||
|
code: (j['code'] as String?) ?? '',
|
||||||
|
message: (j['message'] as String?) ?? '',
|
||||||
|
details: j['details'],
|
||||||
|
detail: j['detail'] as String?,
|
||||||
|
sql: j['sql'] as String?,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// ResolveSpec envelope. [data] is the decoded JSON value (Map, List or scalar).
|
||||||
|
class Response {
|
||||||
|
final bool success;
|
||||||
|
final Object? data;
|
||||||
|
final Metadata? metadata;
|
||||||
|
final ApiError? error;
|
||||||
|
|
||||||
|
const Response({required this.success, this.data, this.metadata, this.error});
|
||||||
|
|
||||||
|
factory Response.fromJson(Map<String, dynamic> j) => Response(
|
||||||
|
success: j['success'] == true,
|
||||||
|
data: j['data'],
|
||||||
|
metadata: j['metadata'] is Map<String, dynamic>
|
||||||
|
? Metadata.fromJson(j['metadata'] as Map<String, dynamic>)
|
||||||
|
: null,
|
||||||
|
error: j['error'] is Map<String, dynamic>
|
||||||
|
? ApiError.fromJson(j['error'] as Map<String, dynamic>)
|
||||||
|
: null,
|
||||||
|
);
|
||||||
|
}
|
||||||
@@ -0,0 +1,14 @@
|
|||||||
|
name: resolvespec
|
||||||
|
description: Client for ResolveSpec (JSON body) and FunctionSpec endpoints.
|
||||||
|
version: 0.1.0
|
||||||
|
publish_to: none
|
||||||
|
|
||||||
|
environment:
|
||||||
|
sdk: ">=3.3.0 <4.0.0"
|
||||||
|
|
||||||
|
dependencies:
|
||||||
|
http: ^1.2.0
|
||||||
|
|
||||||
|
dev_dependencies:
|
||||||
|
lints: ^4.0.0
|
||||||
|
test: ^1.25.0
|
||||||
@@ -0,0 +1,216 @@
|
|||||||
|
import 'dart:convert';
|
||||||
|
|
||||||
|
import 'package:http/http.dart' as http;
|
||||||
|
import 'package:http/testing.dart';
|
||||||
|
import 'package:resolvespec/resolvespec.dart';
|
||||||
|
import 'package:test/test.dart';
|
||||||
|
|
||||||
|
(http.Client, List<http.Request>) stub(int status, Object body,
|
||||||
|
{Map<String, String> headers = const {}}) {
|
||||||
|
final seen = <http.Request>[];
|
||||||
|
final client = MockClient((req) async {
|
||||||
|
seen.add(req);
|
||||||
|
final text = body is String ? body : jsonEncode(body);
|
||||||
|
return http.Response(text, status,
|
||||||
|
headers: {'content-type': 'application/json', ...headers});
|
||||||
|
});
|
||||||
|
return (client, seen);
|
||||||
|
}
|
||||||
|
|
||||||
|
void main() {
|
||||||
|
group('resolvespec', () {
|
||||||
|
test('read posts body with headers', () async {
|
||||||
|
final (c, seen) = stub(200, {
|
||||||
|
'success': true,
|
||||||
|
'data': [
|
||||||
|
{'id': 1}
|
||||||
|
]
|
||||||
|
});
|
||||||
|
final client = ResolveSpecClient(
|
||||||
|
'http://localhost:3000/',
|
||||||
|
ClientOptions(token: 'tok', headers: {'X-Tenant': 'a'}, httpClient: c),
|
||||||
|
);
|
||||||
|
final r = await client.read('public', 'users',
|
||||||
|
options:
|
||||||
|
const Options(limit: 5, filters: [FilterOption('a', 'eq', 1)]));
|
||||||
|
final req = seen.single;
|
||||||
|
expect(req.method, 'POST');
|
||||||
|
expect(req.url.path, '/public/users');
|
||||||
|
expect(req.headers['authorization'], 'Bearer tok');
|
||||||
|
expect(req.headers['x-tenant'], 'a');
|
||||||
|
final body = jsonDecode(req.body) as Map<String, dynamic>;
|
||||||
|
expect(body['operation'], 'read');
|
||||||
|
expect(body['options']['limit'], 5);
|
||||||
|
expect(body.containsKey('id'), isFalse);
|
||||||
|
expect((r.data as List).length, 1);
|
||||||
|
});
|
||||||
|
|
||||||
|
test('id placement', () async {
|
||||||
|
final (c, seen) = stub(200, {'success': true, 'data': {}});
|
||||||
|
final client =
|
||||||
|
ResolveSpecClient('http://x', ClientOptions(httpClient: c));
|
||||||
|
await client.read('s', 'e', id: 7);
|
||||||
|
expect(seen.last.url.path, '/s/e/7');
|
||||||
|
await client.update('s', 'e', {'a': 1}, id: ['1', '2']);
|
||||||
|
expect(seen.last.url.path, '/s/e');
|
||||||
|
final b = jsonDecode(seen.last.body) as Map<String, dynamic>;
|
||||||
|
expect(b['id'], ['1', '2']);
|
||||||
|
expect(b['operation'], 'update');
|
||||||
|
await client.delete('s', 'e', 'a/b');
|
||||||
|
expect(seen.last.url.toString(), 'http://x/s/e/a%2Fb');
|
||||||
|
expect(jsonDecode(seen.last.body)['operation'], 'delete');
|
||||||
|
});
|
||||||
|
|
||||||
|
test('errors', () async {
|
||||||
|
final client = ResolveSpecClient(
|
||||||
|
'http://x',
|
||||||
|
ClientOptions(
|
||||||
|
httpClient: stub(400, {
|
||||||
|
'success': false,
|
||||||
|
'error': {'code': 'x', 'message': 'bad', 'detail': 'why'}
|
||||||
|
}).$1),
|
||||||
|
);
|
||||||
|
await expectLater(
|
||||||
|
client.read('s', 'e'),
|
||||||
|
throwsA(isA<ResolveSpecException>()
|
||||||
|
.having((e) => e.statusCode, 'status', 400)
|
||||||
|
.having((e) => e.error.code, 'code', 'x')
|
||||||
|
.having((e) => e.message, 'message', 'bad')
|
||||||
|
.having((e) => e.error.detail, 'detail', 'why')),
|
||||||
|
);
|
||||||
|
final plain = ResolveSpecClient(
|
||||||
|
'http://x', ClientOptions(httpClient: stub(502, 'bad gateway').$1));
|
||||||
|
await expectLater(
|
||||||
|
plain.read('s', 'e'),
|
||||||
|
throwsA(isA<ResolveSpecException>()
|
||||||
|
.having((e) => e.message, 'message', 'bad gateway')),
|
||||||
|
);
|
||||||
|
final soft = ResolveSpecClient(
|
||||||
|
'http://x',
|
||||||
|
ClientOptions(
|
||||||
|
httpClient: stub(200, {
|
||||||
|
'success': false,
|
||||||
|
'error': {'code': 'c', 'message': 'nope'}
|
||||||
|
}).$1),
|
||||||
|
);
|
||||||
|
await expectLater(
|
||||||
|
soft.read('s', 'e'),
|
||||||
|
throwsA(isA<ResolveSpecException>()
|
||||||
|
.having((e) => e.message, 'message', 'nope')));
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
group('funcspec', () {
|
||||||
|
test('header filters', () {
|
||||||
|
final h = buildHeaders(const FuncSpecOptions(filters: [
|
||||||
|
FilterOption('status', 'eq', 'active'),
|
||||||
|
FilterOption('age', 'gte', 18),
|
||||||
|
FilterOption('name', 'contains', 'x', 'OR'),
|
||||||
|
FilterOption('deleted', 'is_null'),
|
||||||
|
FilterOption('id', 'in', [1, 2]),
|
||||||
|
FilterOption('p', 'between_inclusive', [1, 5]),
|
||||||
|
]));
|
||||||
|
expect(h, {
|
||||||
|
'X-FieldFilter-status': 'active',
|
||||||
|
'X-SearchOp-greaterthanorequal-age': '18',
|
||||||
|
'X-SearchOr-contains-name': 'x',
|
||||||
|
'X-SearchOp-empty-deleted': '',
|
||||||
|
'X-SearchOp-in-id': '1,2',
|
||||||
|
'X-SearchOp-betweeninclusive-p': '1,5',
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
test('misc headers and encoding', () {
|
||||||
|
var h = buildHeaders(const FuncSpecOptions(
|
||||||
|
searchFilters: {'name': 'bob'},
|
||||||
|
customSqlWhere: 'a = 1',
|
||||||
|
customSqlOr: 'b = 2',
|
||||||
|
sort: [SortOption('name', 'asc'), SortOption('created_at', 'DESC')],
|
||||||
|
limit: 5,
|
||||||
|
offset: 10,
|
||||||
|
distinct: true,
|
||||||
|
skipCount: true,
|
||||||
|
skipCache: false,
|
||||||
|
responseFormat: 'syncfusion',
|
||||||
|
));
|
||||||
|
expect(h['X-Sort'], 'name ASC,created_at DESC');
|
||||||
|
expect(h['X-SearchFilter-name'], 'bob');
|
||||||
|
expect(h['X-Custom-SQL-W'], 'a = 1');
|
||||||
|
expect(h['X-SkipCache'], 'false');
|
||||||
|
expect(h['X-Syncfusion'], 'true');
|
||||||
|
|
||||||
|
h = buildHeaders(const FuncSpecOptions(filters: [
|
||||||
|
FilterOption('n', 'eq', 'héllo'),
|
||||||
|
FilterOption('m', 'eq', ' pad'),
|
||||||
|
]));
|
||||||
|
expect(h['X-FieldFilter-n'], startsWith('ZIP_'));
|
||||||
|
expect(decodeHeaderValue(h['X-FieldFilter-n']!), 'héllo');
|
||||||
|
expect(decodeHeaderValue(h['X-FieldFilter-m']!), ' pad');
|
||||||
|
});
|
||||||
|
|
||||||
|
test('query building', () {
|
||||||
|
expect(
|
||||||
|
buildQuery({
|
||||||
|
'a': true,
|
||||||
|
'b': ['x', 'y'],
|
||||||
|
'c': null,
|
||||||
|
'd': 3
|
||||||
|
}),
|
||||||
|
{
|
||||||
|
'a': ['true'],
|
||||||
|
'b': ['x', 'y'],
|
||||||
|
'd': ['3'],
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
test('queryList metadata', () async {
|
||||||
|
final (c, seen) = stub(206, [
|
||||||
|
{'id': 1},
|
||||||
|
{'id': 2}
|
||||||
|
], headers: {
|
||||||
|
'Content-Range': 'items 10-12/50'
|
||||||
|
});
|
||||||
|
final client = FuncSpecClient(
|
||||||
|
'http://x', ClientOptions(token: 'tok', httpClient: c));
|
||||||
|
final r = await client.queryList('/api/users',
|
||||||
|
params: {'org': 1}, options: const FuncSpecOptions(limit: 2));
|
||||||
|
expect(seen.single.method, 'GET');
|
||||||
|
expect(seen.single.url.path, '/api/users');
|
||||||
|
expect(seen.single.url.query, 'org=1');
|
||||||
|
expect(seen.single.headers['x-limit'], '2');
|
||||||
|
expect(
|
||||||
|
r.metadata,
|
||||||
|
const Metadata(
|
||||||
|
total: 50, count: 2, filtered: 50, limit: 2, offset: 10));
|
||||||
|
expect((r.data as List).length, 2);
|
||||||
|
});
|
||||||
|
|
||||||
|
test('query single and error', () async {
|
||||||
|
final ok = FuncSpecClient(
|
||||||
|
'http://x', ClientOptions(httpClient: stub(200, {'id': 1}).$1));
|
||||||
|
final r = await ok.query('api/u');
|
||||||
|
expect(r.metadata, isNull);
|
||||||
|
expect((r.data as Map)['id'], 1);
|
||||||
|
|
||||||
|
final bad = FuncSpecClient(
|
||||||
|
'http://x',
|
||||||
|
ClientOptions(
|
||||||
|
httpClient: stub(400, {
|
||||||
|
'success': false,
|
||||||
|
'error': {
|
||||||
|
'code': 'hook_error',
|
||||||
|
'message': 'Hook execution failed',
|
||||||
|
'detail': 'authentication required'
|
||||||
|
}
|
||||||
|
}).$1),
|
||||||
|
);
|
||||||
|
await expectLater(
|
||||||
|
bad.query('api/u'),
|
||||||
|
throwsA(isA<ResolveSpecException>()
|
||||||
|
.having((e) => e.error.code, 'code', 'hook_error')
|
||||||
|
.having(
|
||||||
|
(e) => e.error.detail, 'detail', 'authentication required')),
|
||||||
|
);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
}
|
||||||
@@ -0,0 +1,40 @@
|
|||||||
|
# resolvespec-go
|
||||||
|
|
||||||
|
Go client for ResolveSpec (JSON body) and FunctionSpec. Module: `github.com/bitechdev/ResolveSpec/clients/resolvespec-go`. Stdlib only.
|
||||||
|
|
||||||
|
## Clients
|
||||||
|
|
||||||
|
| Type | Constructor | Methods |
|
||||||
|
|---|---|---|
|
||||||
|
| `Client` | `NewClient(baseURL, opts...)` | `GetMetadata` `Read` `Create` `Update` `Delete` |
|
||||||
|
| `FuncSpecClient` | `NewFuncSpecClient(baseURL, opts...)` | `Query` `QueryList` `Do` |
|
||||||
|
|
||||||
|
Client options: `WithToken`, `WithHeader`, `WithHTTPClient`. Precedence: Content-Type < custom headers < bearer token.
|
||||||
|
|
||||||
|
## ResolveSpec
|
||||||
|
|
||||||
|
- All methods take `ctx`; `Read`/`Update`/`Delete` take `RecordID` (`nil`, int/string → URL, `[]string` → body).
|
||||||
|
- `Options` fields use pointers for optional ints/bools (`Int(n)`, `Bool(b)`).
|
||||||
|
- Result: `*Response{Success, Data (raw JSON), Metadata}`; `resp.Decode(&v)`.
|
||||||
|
|
||||||
|
## FunctionSpec
|
||||||
|
|
||||||
|
- Routes are server-defined: pass the `path`.
|
||||||
|
- `Params` → query string (slice → repeated keys, bool → `true`/`false`).
|
||||||
|
- `FuncSpecOptions` → `X-*` headers: `Filters`, `SearchFilters`, `CustomSQLWhere`, `CustomSQLOr`, `Sort`, `Limit`, `Offset`, `Distinct`, `SkipCount`, `SkipCache`, `ResponseFormat`.
|
||||||
|
- `QueryList` fills `Metadata` from `Content-Range` (`items a-b/total`); 206 is success.
|
||||||
|
|
||||||
|
## Server quirks
|
||||||
|
|
||||||
|
- `Sort` is raw SQL in ORDER BY (client sends `col ASC|DESC`).
|
||||||
|
- One search operator per column.
|
||||||
|
- Values starting `ZIP_` / `__` are base64-decoded by the server.
|
||||||
|
- Non-ASCII, control chars and edge spaces are auto-encoded (`ZIP_`).
|
||||||
|
|
||||||
|
## Errors
|
||||||
|
|
||||||
|
`*Error{StatusCode, APIError{Code, Message, Detail, SQL}}`.
|
||||||
|
|
||||||
|
## Test
|
||||||
|
|
||||||
|
`go test ./...`
|
||||||
@@ -0,0 +1,121 @@
|
|||||||
|
package resolvespec
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"net/http"
|
||||||
|
"net/url"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
// APIError is the server error object.
|
||||||
|
type APIError struct {
|
||||||
|
Code string `json:"code"`
|
||||||
|
Message string `json:"message"`
|
||||||
|
Details any `json:"details,omitempty"`
|
||||||
|
Detail string `json:"detail,omitempty"` // server-side reason (funcspec / restheadspec)
|
||||||
|
SQL string `json:"sql,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// Error is returned on a non-2xx response or an unsuccessful result.
|
||||||
|
type Error struct {
|
||||||
|
StatusCode int
|
||||||
|
APIError
|
||||||
|
}
|
||||||
|
|
||||||
|
func (e *Error) Error() string {
|
||||||
|
if e.Message != "" {
|
||||||
|
return e.Message
|
||||||
|
}
|
||||||
|
return fmt.Sprintf("http %d", e.StatusCode)
|
||||||
|
}
|
||||||
|
|
||||||
|
type config struct {
|
||||||
|
baseURL string
|
||||||
|
token string
|
||||||
|
headers http.Header
|
||||||
|
http *http.Client
|
||||||
|
}
|
||||||
|
|
||||||
|
// Option configures a client.
|
||||||
|
type Option func(*config)
|
||||||
|
|
||||||
|
func WithToken(token string) Option { return func(c *config) { c.token = token } }
|
||||||
|
func WithHTTPClient(h *http.Client) Option { return func(c *config) { c.http = h } }
|
||||||
|
func WithHeader(name, value string) Option {
|
||||||
|
return func(c *config) { c.headers.Set(name, value) }
|
||||||
|
}
|
||||||
|
|
||||||
|
func newConfig(baseURL string, opts []Option) config {
|
||||||
|
c := config{baseURL: strings.TrimRight(baseURL, "/"), headers: http.Header{}, http: &http.Client{Timeout: 30 * time.Second}}
|
||||||
|
for _, o := range opts {
|
||||||
|
o(&c)
|
||||||
|
}
|
||||||
|
return c
|
||||||
|
}
|
||||||
|
|
||||||
|
// headers: Content-Type < custom headers < bearer token.
|
||||||
|
func (c *config) newRequest(ctx context.Context, method, u string, body []byte) (*http.Request, error) {
|
||||||
|
var r io.Reader
|
||||||
|
if body != nil {
|
||||||
|
r = bytes.NewReader(body)
|
||||||
|
}
|
||||||
|
req, err := http.NewRequestWithContext(ctx, method, u, r)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
req.Header.Set("Content-Type", "application/json")
|
||||||
|
for k, vs := range c.headers {
|
||||||
|
req.Header[k] = append([]string(nil), vs...)
|
||||||
|
}
|
||||||
|
if c.token != "" {
|
||||||
|
req.Header.Set("Authorization", "Bearer "+c.token)
|
||||||
|
}
|
||||||
|
return req, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *config) do(req *http.Request) (*http.Response, []byte, error) {
|
||||||
|
resp, err := c.http.Do(req)
|
||||||
|
if err != nil {
|
||||||
|
return nil, nil, err
|
||||||
|
}
|
||||||
|
defer resp.Body.Close()
|
||||||
|
b, err := io.ReadAll(resp.Body)
|
||||||
|
return resp, b, err
|
||||||
|
}
|
||||||
|
|
||||||
|
func errorFrom(status int, body []byte) *Error {
|
||||||
|
e := &Error{StatusCode: status}
|
||||||
|
var env struct {
|
||||||
|
Error *APIError `json:"error"`
|
||||||
|
}
|
||||||
|
if json.Unmarshal(body, &env) == nil && env.Error != nil {
|
||||||
|
e.APIError = *env.Error
|
||||||
|
}
|
||||||
|
if e.Message == "" {
|
||||||
|
text := ""
|
||||||
|
if !json.Valid(body) {
|
||||||
|
text = strings.TrimSpace(string(body))
|
||||||
|
if len(text) > 200 {
|
||||||
|
text = text[:200]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if text == "" {
|
||||||
|
text = fmt.Sprintf("%s (%d)", http.StatusText(status), status)
|
||||||
|
}
|
||||||
|
e.Message = text
|
||||||
|
}
|
||||||
|
return e
|
||||||
|
}
|
||||||
|
|
||||||
|
func buildURL(base, schema, entity string, id string) string {
|
||||||
|
u := base + "/" + url.PathEscape(schema) + "/" + url.PathEscape(entity)
|
||||||
|
if id != "" {
|
||||||
|
u += "/" + url.PathEscape(id)
|
||||||
|
}
|
||||||
|
return u
|
||||||
|
}
|
||||||
@@ -0,0 +1,274 @@
|
|||||||
|
package resolvespec
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/base64"
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"net/http"
|
||||||
|
"net/url"
|
||||||
|
"regexp"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
"unicode"
|
||||||
|
)
|
||||||
|
|
||||||
|
// FuncSpecOptions are sent to funcspec endpoints as X-* headers.
|
||||||
|
//
|
||||||
|
// Server behaviour (pkg/funcspec): Sort is inserted raw into ORDER BY (so it is sent as SQL
|
||||||
|
// terms); only one search operator per column is kept; values starting with "ZIP_" or "__"
|
||||||
|
// are base64-decoded by the server, so such plaintext values cannot be sent faithfully.
|
||||||
|
type FuncSpecOptions struct {
|
||||||
|
Filters []FilterOption // eq+AND -> X-FieldFilter; others X-SearchOp / X-SearchOr
|
||||||
|
SearchFilters map[string]string // X-SearchFilter-{col}: text ILIKE
|
||||||
|
CustomSQLWhere string // X-Custom-SQL-W
|
||||||
|
CustomSQLOr string // X-Custom-SQL-Or
|
||||||
|
Sort []SortOption
|
||||||
|
Limit *int
|
||||||
|
Offset *int
|
||||||
|
Distinct *bool
|
||||||
|
SkipCount *bool
|
||||||
|
SkipCache *bool
|
||||||
|
ResponseFormat string // simple | detail | syncfusion
|
||||||
|
}
|
||||||
|
|
||||||
|
// Params are query-string values. Slice values are sent as repeated keys (server: IN filter).
|
||||||
|
type Params map[string]any
|
||||||
|
|
||||||
|
// FuncSpecClient calls user-defined SQL endpoints. Routes are defined by the server app.
|
||||||
|
type FuncSpecClient struct{ cfg config }
|
||||||
|
|
||||||
|
func NewFuncSpecClient(baseURL string, opts ...Option) *FuncSpecClient {
|
||||||
|
return &FuncSpecClient{cfg: newConfig(baseURL, opts)}
|
||||||
|
}
|
||||||
|
|
||||||
|
var operatorMap = map[string]string{
|
||||||
|
"eq": "equals", "neq": "notequals", "gt": "greaterthan", "gte": "greaterthanorequal",
|
||||||
|
"lt": "lessthan", "lte": "lessthanorequal", "like": "contains", "ilike": "contains",
|
||||||
|
"contains": "contains", "startswith": "beginswith", "endswith": "endswith", "in": "in",
|
||||||
|
"between": "between", "between_inclusive": "betweeninclusive",
|
||||||
|
"is_null": "empty", "is_not_null": "notempty",
|
||||||
|
}
|
||||||
|
|
||||||
|
func scalar(v any) string {
|
||||||
|
switch x := v.(type) {
|
||||||
|
case nil:
|
||||||
|
return ""
|
||||||
|
case bool:
|
||||||
|
return strconv.FormatBool(x)
|
||||||
|
case string:
|
||||||
|
return x
|
||||||
|
case fmt.Stringer:
|
||||||
|
return x.String()
|
||||||
|
}
|
||||||
|
return fmt.Sprint(v)
|
||||||
|
}
|
||||||
|
|
||||||
|
func filterValue(v any) string {
|
||||||
|
switch x := v.(type) {
|
||||||
|
case nil:
|
||||||
|
return ""
|
||||||
|
case []string:
|
||||||
|
return strings.Join(x, ",")
|
||||||
|
case []int:
|
||||||
|
parts := make([]string, len(x))
|
||||||
|
for i, n := range x {
|
||||||
|
parts[i] = strconv.Itoa(n)
|
||||||
|
}
|
||||||
|
return strings.Join(parts, ",")
|
||||||
|
case []any:
|
||||||
|
parts := make([]string, len(x))
|
||||||
|
for i, n := range x {
|
||||||
|
parts[i] = scalar(n)
|
||||||
|
}
|
||||||
|
return strings.Join(parts, ",")
|
||||||
|
}
|
||||||
|
return scalar(v)
|
||||||
|
}
|
||||||
|
|
||||||
|
// EncodeHeaderValue base64-encodes (UTF-8) with the ZIP_ prefix.
|
||||||
|
func EncodeHeaderValue(v string) string { return "ZIP_" + base64.StdEncoding.EncodeToString([]byte(v)) }
|
||||||
|
|
||||||
|
// DecodeHeaderValue decodes a value that may carry a ZIP_ or __ prefix (nested allowed).
|
||||||
|
func DecodeHeaderValue(v string) string {
|
||||||
|
for _, p := range []string{"ZIP_", "__"} {
|
||||||
|
if strings.HasPrefix(v, p) {
|
||||||
|
b64 := strings.NewReplacer("\n", "", "\r", "", " ", "").Replace(v[len(p):])
|
||||||
|
for len(b64)%4 != 0 {
|
||||||
|
b64 += "="
|
||||||
|
}
|
||||||
|
raw, err := base64.StdEncoding.DecodeString(b64)
|
||||||
|
if err != nil {
|
||||||
|
return v
|
||||||
|
}
|
||||||
|
return DecodeHeaderValue(string(raw))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return v
|
||||||
|
}
|
||||||
|
|
||||||
|
// safe encodes values that are unsafe as raw header/query text (non-ASCII, control chars, edge spaces).
|
||||||
|
func safe(v string) string {
|
||||||
|
if v != strings.TrimSpace(v) {
|
||||||
|
return EncodeHeaderValue(v)
|
||||||
|
}
|
||||||
|
for _, r := range v {
|
||||||
|
if r > unicode.MaxASCII || !unicode.IsPrint(r) {
|
||||||
|
return EncodeHeaderValue(v)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return v
|
||||||
|
}
|
||||||
|
|
||||||
|
// BuildHeaders builds the X-* headers understood by funcspec.ParseParameters.
|
||||||
|
func BuildHeaders(o *FuncSpecOptions) map[string]string {
|
||||||
|
h := map[string]string{}
|
||||||
|
if o == nil {
|
||||||
|
return h
|
||||||
|
}
|
||||||
|
for _, f := range o.Filters {
|
||||||
|
logic := f.LogicOperator
|
||||||
|
if logic == "" {
|
||||||
|
logic = "AND"
|
||||||
|
}
|
||||||
|
v := safe(filterValue(f.Value))
|
||||||
|
if f.Operator == "eq" && logic == "AND" {
|
||||||
|
h["X-FieldFilter-"+f.Column] = v
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
op := operatorMap[f.Operator]
|
||||||
|
if op == "" {
|
||||||
|
op = f.Operator
|
||||||
|
}
|
||||||
|
kind := "X-SearchOp"
|
||||||
|
if logic == "OR" {
|
||||||
|
kind = "X-SearchOr"
|
||||||
|
}
|
||||||
|
h[kind+"-"+op+"-"+f.Column] = v
|
||||||
|
}
|
||||||
|
for col, text := range o.SearchFilters {
|
||||||
|
h["X-SearchFilter-"+col] = safe(text)
|
||||||
|
}
|
||||||
|
if o.CustomSQLWhere != "" {
|
||||||
|
h["X-Custom-SQL-W"] = safe(o.CustomSQLWhere)
|
||||||
|
}
|
||||||
|
if o.CustomSQLOr != "" {
|
||||||
|
h["X-Custom-SQL-Or"] = safe(o.CustomSQLOr)
|
||||||
|
}
|
||||||
|
if len(o.Sort) > 0 {
|
||||||
|
terms := make([]string, len(o.Sort))
|
||||||
|
for i, s := range o.Sort {
|
||||||
|
dir := "ASC"
|
||||||
|
if strings.EqualFold(s.Direction, "desc") {
|
||||||
|
dir = "DESC"
|
||||||
|
}
|
||||||
|
terms[i] = s.Column + " " + dir // funcspec puts this verbatim into ORDER BY
|
||||||
|
}
|
||||||
|
h["X-Sort"] = safe(strings.Join(terms, ","))
|
||||||
|
}
|
||||||
|
if o.Limit != nil {
|
||||||
|
h["X-Limit"] = strconv.Itoa(*o.Limit)
|
||||||
|
}
|
||||||
|
if o.Offset != nil {
|
||||||
|
h["X-Offset"] = strconv.Itoa(*o.Offset)
|
||||||
|
}
|
||||||
|
for name, v := range map[string]*bool{"X-Distinct": o.Distinct, "X-SkipCount": o.SkipCount, "X-SkipCache": o.SkipCache} {
|
||||||
|
if v != nil {
|
||||||
|
h[name] = strconv.FormatBool(*v)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
switch o.ResponseFormat {
|
||||||
|
case "simple":
|
||||||
|
h["X-SimpleApi"] = "true"
|
||||||
|
case "detail":
|
||||||
|
h["X-DetailApi"] = "true"
|
||||||
|
case "syncfusion":
|
||||||
|
h["X-Syncfusion"] = "true"
|
||||||
|
}
|
||||||
|
return h
|
||||||
|
}
|
||||||
|
|
||||||
|
// BuildQuery builds query-string values: bools -> true/false, slices -> repeated keys, nil skipped.
|
||||||
|
func BuildQuery(p Params) url.Values {
|
||||||
|
q := url.Values{}
|
||||||
|
for k, v := range p {
|
||||||
|
switch x := v.(type) {
|
||||||
|
case nil:
|
||||||
|
case []string:
|
||||||
|
for _, e := range x {
|
||||||
|
q.Add(k, safe(e))
|
||||||
|
}
|
||||||
|
case []int:
|
||||||
|
for _, e := range x {
|
||||||
|
q.Add(k, strconv.Itoa(e))
|
||||||
|
}
|
||||||
|
case []any:
|
||||||
|
for _, e := range x {
|
||||||
|
q.Add(k, safe(scalar(e)))
|
||||||
|
}
|
||||||
|
default:
|
||||||
|
q.Add(k, safe(scalar(v)))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return q
|
||||||
|
}
|
||||||
|
|
||||||
|
var contentRange = regexp.MustCompile(`(\d+)-(\d+)/(\d+)`)
|
||||||
|
|
||||||
|
func metadata(h http.Header, o *FuncSpecOptions) *Metadata {
|
||||||
|
m := &Metadata{}
|
||||||
|
if g := contentRange.FindStringSubmatch(h.Get("Content-Range")); g != nil {
|
||||||
|
start, _ := strconv.ParseInt(g[1], 10, 64)
|
||||||
|
end, _ := strconv.ParseInt(g[2], 10, 64)
|
||||||
|
total, _ := strconv.ParseInt(g[3], 10, 64)
|
||||||
|
m.Total, m.Count, m.Filtered, m.Offset = total, end-start, total, int(start)
|
||||||
|
}
|
||||||
|
if o != nil && o.Limit != nil {
|
||||||
|
m.Limit = *o.Limit
|
||||||
|
}
|
||||||
|
return m
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *FuncSpecClient) call(ctx context.Context, method, path string, p Params, o *FuncSpecOptions, withMeta bool) (*Response, error) {
|
||||||
|
u := c.cfg.baseURL + "/" + strings.TrimLeft(path, "/")
|
||||||
|
if q := BuildQuery(p); len(q) > 0 {
|
||||||
|
u += "?" + q.Encode()
|
||||||
|
}
|
||||||
|
req, err := c.cfg.newRequest(ctx, strings.ToUpper(method), u, nil)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
for k, v := range BuildHeaders(o) {
|
||||||
|
req.Header.Set(k, v)
|
||||||
|
}
|
||||||
|
resp, b, err := c.cfg.do(req)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if resp.StatusCode < 200 || resp.StatusCode > 299 { // 206 is success
|
||||||
|
return nil, errorFrom(resp.StatusCode, b)
|
||||||
|
}
|
||||||
|
out := &Response{Success: true, Data: json.RawMessage(b)}
|
||||||
|
if len(b) == 0 {
|
||||||
|
out.Data = json.RawMessage("null")
|
||||||
|
}
|
||||||
|
if withMeta {
|
||||||
|
out.Metadata = metadata(resp.Header, o)
|
||||||
|
}
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Query calls a single-record endpoint (SqlQuery). Data is the row object.
|
||||||
|
func (c *FuncSpecClient) Query(ctx context.Context, path string, p Params, o *FuncSpecOptions) (*Response, error) {
|
||||||
|
return c.call(ctx, http.MethodGet, path, p, o, false)
|
||||||
|
}
|
||||||
|
|
||||||
|
// QueryList calls a list endpoint (SqlQueryList). Metadata comes from Content-Range.
|
||||||
|
func (c *FuncSpecClient) QueryList(ctx context.Context, path string, p Params, o *FuncSpecOptions) (*Response, error) {
|
||||||
|
return c.call(ctx, http.MethodGet, path, p, o, true)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Do is like Query/QueryList with an explicit HTTP method (routes are app-defined).
|
||||||
|
func (c *FuncSpecClient) Do(ctx context.Context, method, path string, p Params, o *FuncSpecOptions, list bool) (*Response, error) {
|
||||||
|
return c.call(ctx, method, path, p, o, list)
|
||||||
|
}
|
||||||
@@ -0,0 +1,98 @@
|
|||||||
|
package resolvespec
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"reflect"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestBuildHeadersFilters(t *testing.T) {
|
||||||
|
got := BuildHeaders(&FuncSpecOptions{Filters: []FilterOption{
|
||||||
|
{Column: "status", Operator: "eq", Value: "active"},
|
||||||
|
{Column: "age", Operator: "gte", Value: 18},
|
||||||
|
{Column: "name", Operator: "contains", Value: "x", LogicOperator: "OR"},
|
||||||
|
{Column: "deleted", Operator: "is_null"},
|
||||||
|
{Column: "id", Operator: "in", Value: []int{1, 2}},
|
||||||
|
{Column: "p", Operator: "between_inclusive", Value: []any{1, 5}},
|
||||||
|
}})
|
||||||
|
want := map[string]string{
|
||||||
|
"X-FieldFilter-status": "active",
|
||||||
|
"X-SearchOp-greaterthanorequal-age": "18",
|
||||||
|
"X-SearchOr-contains-name": "x",
|
||||||
|
"X-SearchOp-empty-deleted": "",
|
||||||
|
"X-SearchOp-in-id": "1,2",
|
||||||
|
"X-SearchOp-betweeninclusive-p": "1,5",
|
||||||
|
}
|
||||||
|
if !reflect.DeepEqual(got, want) {
|
||||||
|
t.Fatalf("%v", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestBuildHeadersMisc(t *testing.T) {
|
||||||
|
got := BuildHeaders(&FuncSpecOptions{
|
||||||
|
SearchFilters: map[string]string{"name": "bob"}, CustomSQLWhere: "a = 1", CustomSQLOr: "b = 2",
|
||||||
|
Sort: []SortOption{{"name", "asc"}, {"created_at", "DESC"}},
|
||||||
|
Limit: Int(5), Offset: Int(10), Distinct: Bool(true), SkipCount: Bool(true), SkipCache: Bool(false),
|
||||||
|
ResponseFormat: "syncfusion",
|
||||||
|
})
|
||||||
|
want := map[string]string{
|
||||||
|
"X-SearchFilter-name": "bob", "X-Custom-SQL-W": "a = 1", "X-Custom-SQL-Or": "b = 2",
|
||||||
|
"X-Sort": "name ASC,created_at DESC", "X-Limit": "5", "X-Offset": "10", "X-Distinct": "true",
|
||||||
|
"X-SkipCount": "true", "X-SkipCache": "false", "X-Syncfusion": "true",
|
||||||
|
}
|
||||||
|
if !reflect.DeepEqual(got, want) {
|
||||||
|
t.Fatalf("%v", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestEncodeUnsafe(t *testing.T) {
|
||||||
|
h := BuildHeaders(&FuncSpecOptions{Filters: []FilterOption{{Column: "n", Operator: "eq", Value: "héllo"}, {Column: "m", Operator: "eq", Value: " pad"}}})
|
||||||
|
for _, k := range []string{"X-FieldFilter-n", "X-FieldFilter-m"} {
|
||||||
|
if len(h[k]) < 4 || h[k][:4] != "ZIP_" {
|
||||||
|
t.Fatalf("%s=%q", k, h[k])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if DecodeHeaderValue(h["X-FieldFilter-n"]) != "héllo" || DecodeHeaderValue(h["X-FieldFilter-m"]) != " pad" {
|
||||||
|
t.Fatal("roundtrip")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestBuildQuery(t *testing.T) {
|
||||||
|
q := BuildQuery(Params{"a": true, "b": []string{"x", "y"}, "c": nil, "d": 3})
|
||||||
|
if q.Get("a") != "true" || !reflect.DeepEqual(q["b"], []string{"x", "y"}) || q.Has("c") || q.Get("d") != "3" {
|
||||||
|
t.Fatalf("%v", q)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestQueryListMetadata(t *testing.T) {
|
||||||
|
srv, s := server(t, 206, `[{"id":1},{"id":2}]`, map[string]string{"Content-Range": "items 10-12/50"})
|
||||||
|
c := NewFuncSpecClient(srv.URL, WithToken("tok"))
|
||||||
|
resp, err := c.QueryList(context.Background(), "/api/users", Params{"org": 1}, &FuncSpecOptions{Limit: Int(2)})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if s.method != "GET" || s.path != "/api/users?org=1" || s.header.Get("X-Limit") != "2" {
|
||||||
|
t.Fatalf("%s %v", s.path, s.header)
|
||||||
|
}
|
||||||
|
m := resp.Metadata
|
||||||
|
if m.Total != 50 || m.Count != 2 || m.Offset != 10 || m.Limit != 2 || m.Filtered != 50 {
|
||||||
|
t.Fatalf("%+v", m)
|
||||||
|
}
|
||||||
|
var rows []map[string]any
|
||||||
|
if err := resp.Decode(&rows); err != nil || len(rows) != 2 {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestQuerySingleNoMetadataAndError(t *testing.T) {
|
||||||
|
srv, _ := server(t, 200, `{"id":1}`, nil)
|
||||||
|
resp, err := NewFuncSpecClient(srv.URL).Query(context.Background(), "api/u", nil, nil)
|
||||||
|
if err != nil || resp.Metadata != nil {
|
||||||
|
t.Fatalf("%v %v", resp, err)
|
||||||
|
}
|
||||||
|
srv2, _ := server(t, 400, `{"success":false,"error":{"code":"hook_error","message":"Hook execution failed","detail":"authentication required"}}`, nil)
|
||||||
|
_, err = NewFuncSpecClient(srv2.URL).Query(context.Background(), "api/u", nil, nil)
|
||||||
|
if e := err.(*Error); e.Code != "hook_error" || e.Detail != "authentication required" {
|
||||||
|
t.Fatalf("%#v", e)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,3 @@
|
|||||||
|
module github.com/bitechdev/ResolveSpec/clients/resolvespec-go
|
||||||
|
|
||||||
|
go 1.22
|
||||||
@@ -0,0 +1,94 @@
|
|||||||
|
package resolvespec
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"net/http"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Client speaks the ResolveSpec JSON body protocol: POST {operation, data, options}.
|
||||||
|
type Client struct{ cfg config }
|
||||||
|
|
||||||
|
func NewClient(baseURL string, opts ...Option) *Client {
|
||||||
|
return &Client{cfg: newConfig(baseURL, opts)}
|
||||||
|
}
|
||||||
|
|
||||||
|
// RecordID is a single id (int or string, sent in the URL) or a []string (sent in the body).
|
||||||
|
type RecordID any
|
||||||
|
|
||||||
|
func urlID(id RecordID) string {
|
||||||
|
switch v := id.(type) {
|
||||||
|
case nil:
|
||||||
|
return ""
|
||||||
|
case []string:
|
||||||
|
return ""
|
||||||
|
case string:
|
||||||
|
return v
|
||||||
|
default:
|
||||||
|
return fmt.Sprint(v)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func bodyID(id RecordID) []string {
|
||||||
|
ids, _ := id.([]string)
|
||||||
|
return ids
|
||||||
|
}
|
||||||
|
|
||||||
|
type request struct {
|
||||||
|
Operation string `json:"operation"`
|
||||||
|
ID []string `json:"id,omitempty"`
|
||||||
|
Data any `json:"data,omitempty"`
|
||||||
|
Options *Options `json:"options,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *Client) send(ctx context.Context, method, schema, entity, id string, body any) (*Response, error) {
|
||||||
|
var payload []byte
|
||||||
|
if body != nil {
|
||||||
|
var err error
|
||||||
|
if payload, err = json.Marshal(body); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
req, err := c.cfg.newRequest(ctx, method, buildURL(c.cfg.baseURL, schema, entity, id), payload)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
resp, b, err := c.cfg.do(req)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if resp.StatusCode < 200 || resp.StatusCode > 299 {
|
||||||
|
return nil, errorFrom(resp.StatusCode, b)
|
||||||
|
}
|
||||||
|
var out Response
|
||||||
|
if err := json.Unmarshal(b, &out); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if !out.Success && out.Error != nil {
|
||||||
|
return nil, &Error{StatusCode: resp.StatusCode, APIError: *out.Error}
|
||||||
|
}
|
||||||
|
return &out, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// GetMetadata returns table metadata (GET /{schema}/{entity}).
|
||||||
|
func (c *Client) GetMetadata(ctx context.Context, schema, entity string) (*Response, error) {
|
||||||
|
return c.send(ctx, http.MethodGet, schema, entity, "", nil)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Read reads records; id may be nil, an int/string (URL) or []string (body).
|
||||||
|
func (c *Client) Read(ctx context.Context, schema, entity string, id RecordID, opts *Options) (*Response, error) {
|
||||||
|
return c.send(ctx, http.MethodPost, schema, entity, urlID(id), request{Operation: "read", ID: bodyID(id), Options: opts})
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *Client) Create(ctx context.Context, schema, entity string, data any, opts *Options) (*Response, error) {
|
||||||
|
return c.send(ctx, http.MethodPost, schema, entity, "", request{Operation: "create", Data: data, Options: opts})
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *Client) Update(ctx context.Context, schema, entity string, data any, id RecordID, opts *Options) (*Response, error) {
|
||||||
|
return c.send(ctx, http.MethodPost, schema, entity, urlID(id), request{Operation: "update", ID: bodyID(id), Data: data, Options: opts})
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *Client) Delete(ctx context.Context, schema, entity string, id RecordID) (*Response, error) {
|
||||||
|
return c.send(ctx, http.MethodPost, schema, entity, urlID(id), request{Operation: "delete"})
|
||||||
|
}
|
||||||
@@ -0,0 +1,99 @@
|
|||||||
|
package resolvespec
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"io"
|
||||||
|
"net/http"
|
||||||
|
"net/http/httptest"
|
||||||
|
"reflect"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
type seen struct {
|
||||||
|
method, path string
|
||||||
|
header http.Header
|
||||||
|
body map[string]any
|
||||||
|
}
|
||||||
|
|
||||||
|
func server(t *testing.T, status int, body string, hdr map[string]string) (*httptest.Server, *seen) {
|
||||||
|
t.Helper()
|
||||||
|
s := &seen{}
|
||||||
|
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
s.method, s.path, s.header = r.Method, r.URL.EscapedPath()+"?"+r.URL.RawQuery, r.Header
|
||||||
|
b, _ := io.ReadAll(r.Body)
|
||||||
|
if len(b) > 0 {
|
||||||
|
_ = json.Unmarshal(b, &s.body)
|
||||||
|
}
|
||||||
|
for k, v := range hdr {
|
||||||
|
w.Header().Set(k, v)
|
||||||
|
}
|
||||||
|
w.WriteHeader(status)
|
||||||
|
_, _ = w.Write([]byte(body))
|
||||||
|
}))
|
||||||
|
t.Cleanup(srv.Close)
|
||||||
|
return srv, s
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestReadBody(t *testing.T) {
|
||||||
|
srv, s := server(t, 200, `{"success":true,"data":[{"id":1}]}`, nil)
|
||||||
|
c := NewClient(srv.URL+"/", WithToken("tok"), WithHeader("X-Tenant", "a"))
|
||||||
|
resp, err := c.Read(context.Background(), "public", "users", nil, &Options{Limit: Int(5), Filters: []FilterOption{{Column: "a", Operator: "eq", Value: 1}}})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if s.method != "POST" || s.path != "/public/users?" {
|
||||||
|
t.Fatalf("got %s %s", s.method, s.path)
|
||||||
|
}
|
||||||
|
if s.header.Get("Authorization") != "Bearer tok" || s.header.Get("X-Tenant") != "a" {
|
||||||
|
t.Fatalf("headers %v", s.header)
|
||||||
|
}
|
||||||
|
if s.body["operation"] != "read" || s.body["options"].(map[string]any)["limit"] != float64(5) {
|
||||||
|
t.Fatalf("body %v", s.body)
|
||||||
|
}
|
||||||
|
var rows []map[string]any
|
||||||
|
if err := resp.Decode(&rows); err != nil || len(rows) != 1 {
|
||||||
|
t.Fatalf("decode %v %v", rows, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestIDPlacement(t *testing.T) {
|
||||||
|
srv, s := server(t, 200, `{"success":true,"data":{}}`, nil)
|
||||||
|
c := NewClient(srv.URL)
|
||||||
|
ctx := context.Background()
|
||||||
|
_, _ = c.Read(ctx, "s", "e", 7, nil)
|
||||||
|
if s.path != "/s/e/7?" || s.body["id"] != nil {
|
||||||
|
t.Fatalf("%s %v", s.path, s.body)
|
||||||
|
}
|
||||||
|
_, _ = c.Update(ctx, "s", "e", map[string]any{"a": 1}, []string{"1", "2"}, nil)
|
||||||
|
if s.path != "/s/e?" || !reflect.DeepEqual(s.body["id"], []any{"1", "2"}) || s.body["operation"] != "update" {
|
||||||
|
t.Fatalf("%s %v", s.path, s.body)
|
||||||
|
}
|
||||||
|
_, _ = c.Delete(ctx, "s", "e", "a/b")
|
||||||
|
if s.path != "/s/e/a%2Fb?" || s.body["operation"] != "delete" {
|
||||||
|
t.Fatalf("%s %v", s.path, s.body)
|
||||||
|
}
|
||||||
|
_, _ = c.GetMetadata(ctx, "s", "e")
|
||||||
|
if s.method != "GET" {
|
||||||
|
t.Fatal(s.method)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestErrors(t *testing.T) {
|
||||||
|
srv, _ := server(t, 400, `{"success":false,"error":{"code":"x","message":"bad","detail":"why"}}`, nil)
|
||||||
|
_, err := NewClient(srv.URL).Read(context.Background(), "s", "e", nil, nil)
|
||||||
|
e, ok := err.(*Error)
|
||||||
|
if !ok || e.StatusCode != 400 || e.Code != "x" || e.Message != "bad" || e.Detail != "why" {
|
||||||
|
t.Fatalf("%#v", err)
|
||||||
|
}
|
||||||
|
srv2, _ := server(t, 502, "bad gateway", nil)
|
||||||
|
_, err = NewClient(srv2.URL).Read(context.Background(), "s", "e", nil, nil)
|
||||||
|
if e := err.(*Error); e.StatusCode != 502 || e.Message != "bad gateway" {
|
||||||
|
t.Fatalf("%#v", e)
|
||||||
|
}
|
||||||
|
srv3, _ := server(t, 200, `{"success":false,"error":{"code":"c","message":"nope"}}`, nil)
|
||||||
|
_, err = NewClient(srv3.URL).Read(context.Background(), "s", "e", nil, nil)
|
||||||
|
if e := err.(*Error); e.Message != "nope" {
|
||||||
|
t.Fatalf("%#v", e)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,107 @@
|
|||||||
|
// Package resolvespec is a client for ResolveSpec (JSON body) and FunctionSpec endpoints.
|
||||||
|
package resolvespec
|
||||||
|
|
||||||
|
import "encoding/json"
|
||||||
|
|
||||||
|
// FilterOption mirrors common.FilterOption. Operator: eq neq gt gte lt lte like ilike in
|
||||||
|
// contains startswith endswith between between_inclusive is_null is_not_null.
|
||||||
|
type FilterOption struct {
|
||||||
|
Column string `json:"column"`
|
||||||
|
Operator string `json:"operator"`
|
||||||
|
Value any `json:"value"`
|
||||||
|
LogicOperator string `json:"logic_operator,omitempty"` // AND | OR
|
||||||
|
}
|
||||||
|
|
||||||
|
type SortOption struct {
|
||||||
|
Column string `json:"column"`
|
||||||
|
Direction string `json:"direction"` // asc | desc
|
||||||
|
}
|
||||||
|
|
||||||
|
type Parameter struct {
|
||||||
|
Name string `json:"name"`
|
||||||
|
Value string `json:"value"`
|
||||||
|
Sequence int `json:"sequence,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type CustomOperator struct {
|
||||||
|
Name string `json:"name"`
|
||||||
|
SQL string `json:"sql"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type ComputedColumn struct {
|
||||||
|
Name string `json:"name"`
|
||||||
|
Expression string `json:"expression"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type PreloadOption struct {
|
||||||
|
Relation string `json:"relation,omitempty"`
|
||||||
|
TableName string `json:"table_name,omitempty"`
|
||||||
|
Columns []string `json:"columns,omitempty"`
|
||||||
|
OmitColumns []string `json:"omit_columns,omitempty"`
|
||||||
|
Sort []SortOption `json:"sort,omitempty"`
|
||||||
|
Filters []FilterOption `json:"filters,omitempty"`
|
||||||
|
Where string `json:"where,omitempty"`
|
||||||
|
Limit *int `json:"limit,omitempty"`
|
||||||
|
Offset *int `json:"offset,omitempty"`
|
||||||
|
Updateable *bool `json:"updateable,omitempty"`
|
||||||
|
ComputedQL map[string]string `json:"computed_ql,omitempty"`
|
||||||
|
Recursive bool `json:"recursive,omitempty"`
|
||||||
|
PrimaryKey string `json:"primary_key,omitempty"`
|
||||||
|
RelatedKey string `json:"related_key,omitempty"`
|
||||||
|
ForeignKey string `json:"foreign_key,omitempty"`
|
||||||
|
RecursiveChildKey string `json:"recursive_child_key,omitempty"`
|
||||||
|
SQLJoins []string `json:"sql_joins,omitempty"`
|
||||||
|
JoinAliases []string `json:"join_aliases,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type VectorSearchOption struct {
|
||||||
|
Column string `json:"column"`
|
||||||
|
Vector []float64 `json:"vector"`
|
||||||
|
Metric string `json:"metric,omitempty"` // l2 (default) | cosine | ip
|
||||||
|
As string `json:"as,omitempty"` // distance alias, default _distance
|
||||||
|
Direction string `json:"direction,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// Options is the ResolveSpec request options object.
|
||||||
|
type Options struct {
|
||||||
|
Preload []PreloadOption `json:"preload,omitempty"`
|
||||||
|
Columns []string `json:"columns,omitempty"`
|
||||||
|
OmitColumns []string `json:"omit_columns,omitempty"`
|
||||||
|
Filters []FilterOption `json:"filters,omitempty"`
|
||||||
|
Sort []SortOption `json:"sort,omitempty"`
|
||||||
|
Limit *int `json:"limit,omitempty"`
|
||||||
|
Offset *int `json:"offset,omitempty"`
|
||||||
|
CustomOperators []CustomOperator `json:"customOperators,omitempty"`
|
||||||
|
ComputedColumns []ComputedColumn `json:"computedColumns,omitempty"`
|
||||||
|
Parameters []Parameter `json:"parameters,omitempty"`
|
||||||
|
CursorForward string `json:"cursor_forward,omitempty"`
|
||||||
|
CursorBackward string `json:"cursor_backward,omitempty"`
|
||||||
|
FetchRowNumber string `json:"fetch_row_number,omitempty"`
|
||||||
|
VectorSearch *VectorSearchOption `json:"vector_search,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// Metadata of a list response.
|
||||||
|
type Metadata struct {
|
||||||
|
Total int64 `json:"total"`
|
||||||
|
Count int64 `json:"count"`
|
||||||
|
Filtered int64 `json:"filtered"`
|
||||||
|
Limit int `json:"limit"`
|
||||||
|
Offset int `json:"offset"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// Response is the ResolveSpec envelope. Data is left raw for the caller to decode.
|
||||||
|
type Response struct {
|
||||||
|
Success bool `json:"success"`
|
||||||
|
Data json.RawMessage `json:"data"`
|
||||||
|
Metadata *Metadata `json:"metadata,omitempty"`
|
||||||
|
Error *APIError `json:"error,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// Decode unmarshals Data into v.
|
||||||
|
func (r *Response) Decode(v any) error { return json.Unmarshal(r.Data, v) }
|
||||||
|
|
||||||
|
// Int returns a pointer to n, for optional Options fields.
|
||||||
|
func Int(n int) *int { return &n }
|
||||||
|
|
||||||
|
// Bool returns a pointer to b.
|
||||||
|
func Bool(b bool) *bool { return &b }
|
||||||
@@ -0,0 +1,14 @@
|
|||||||
|
# @warkypublic/resolvespec-js
|
||||||
|
|
||||||
|
## 1.0.2
|
||||||
|
|
||||||
|
### Patch Changes
|
||||||
|
|
||||||
|
- b587cbd: Forward custom ClientConfig headers on every ResolveSpec and HeaderSpec request. Merge headers case-insensitively and isolate cached clients by URL and effective headers, including authentication and tenant headers.
|
||||||
|
- 7f8982f: fix: added headers and few fixes
|
||||||
|
|
||||||
|
## 1.0.1
|
||||||
|
|
||||||
|
### Patch Changes
|
||||||
|
|
||||||
|
- Fixed headerpsec
|
||||||
@@ -28,7 +28,7 @@ import { ResolveSpecClient, getResolveSpecClient } from '@warkypublic/resolvespe
|
|||||||
// Class instantiation
|
// Class instantiation
|
||||||
const client = new ResolveSpecClient({ baseUrl: 'http://localhost:3000', token: 'your-token' });
|
const client = new ResolveSpecClient({ baseUrl: 'http://localhost:3000', token: 'your-token' });
|
||||||
|
|
||||||
// Or singleton factory (returns cached instance per baseUrl)
|
// Or singleton factory (returns cached instance per baseUrl and effective headers)
|
||||||
const client = getResolveSpecClient({ baseUrl: 'http://localhost:3000', token: 'your-token' });
|
const client = getResolveSpecClient({ baseUrl: 'http://localhost:3000', token: 'your-token' });
|
||||||
|
|
||||||
// Read with filters, sort, pagination
|
// Read with filters, sort, pagination
|
||||||
@@ -106,6 +106,25 @@ await client.delete('public', 'users', '42');
|
|||||||
| `X-Fetch-RowNumber` | `fetch_row_number` | string |
|
| `X-Fetch-RowNumber` | `fetch_row_number` | string |
|
||||||
| `X-CQL-SEL-{col}` | `computedColumns` | expression |
|
| `X-CQL-SEL-{col}` | `computedColumns` | expression |
|
||||||
| `X-Custom-SQL-W` | `customOperators` | SQL AND-joined |
|
| `X-Custom-SQL-W` | `customOperators` | SQL AND-joined |
|
||||||
|
| `X-Preload-Where` | `preload[].where` | applies to all preloads in `X-Preload`; differing wheres go to `X-Preload-{n}` + `X-Preload-{n}-Where` |
|
||||||
|
| `X-Expand` | `expand` | `Rel:col1,col2` pipe-separated (LEFT JOIN) |
|
||||||
|
| `X-Custom-SQL-Join` | `custom_sql_joins` | JOIN clauses, pipe-separated |
|
||||||
|
| `X-Custom-SQL-Or` | `custom_sql_or` | SQL OR-joined |
|
||||||
|
| `X-SearchCols` | `search_columns` | comma-separated |
|
||||||
|
| `X-AdvSQL-{col}` | `advanced_sql` | column -> SQL |
|
||||||
|
| `X-SpatialFilter-{col}` | `filters` (`st_dwithin`, `st_*`, `bbox`) | JSON `{op,value,logic}` |
|
||||||
|
| `X-VectorFilter-{col}` | `filters` (`l2_within`, `cosine_within`, `ip_within`) | JSON `{op,value,logic}` |
|
||||||
|
| `X-Vector-Search-{col}` / `-Vector` / `-As` / `-Dir` | `vector_search` | metric / JSON array / alias / asc\|desc |
|
||||||
|
| `X-Clean-JSON` | `clean_json` | bool |
|
||||||
|
| `X-Distinct` | `distinct` | bool |
|
||||||
|
| `X-SkipCount` / `X-SkipCache` | `skip_count` / `skip_cache` | bool |
|
||||||
|
| `X-PKRow` | `pk_row` | string |
|
||||||
|
| `X-SimpleApi` / `X-DetailApi` / `X-Syncfusion` | `response_format` | `simple` \| `detail` \| `syncfusion` |
|
||||||
|
| `X-Single-Record-As-Object` | `single_record_as_object` | bool (server default true) |
|
||||||
|
| `X-Transaction-Atomic` | `atomic_transaction` | bool |
|
||||||
|
| `X-Files` | `xfiles` | JSON, sent as `ZIP_` base64 |
|
||||||
|
|
||||||
|
Extended fields live on `HeaderSpecOptions` (extends `Options`); `vector_search` is on `Options`.
|
||||||
|
|
||||||
### Utility Functions
|
### Utility Functions
|
||||||
|
|
||||||
@@ -211,3 +230,25 @@ pnpm run lint # eslint
|
|||||||
## License
|
## License
|
||||||
|
|
||||||
MIT
|
MIT
|
||||||
|
|
||||||
|
### Custom HTTP headers
|
||||||
|
|
||||||
|
Both `ResolveSpecClient` and `HeaderSpecClient` (including their factory functions)
|
||||||
|
accept `headers` in `ClientConfig` and send them on every HTTP request:
|
||||||
|
|
||||||
|
```typescript
|
||||||
|
const client = new ResolveSpecClient({
|
||||||
|
baseUrl: 'http://localhost:3000',
|
||||||
|
token: 'your-token',
|
||||||
|
headers: { 'X-Tenant': 'acme' },
|
||||||
|
});
|
||||||
|
```
|
||||||
|
|
||||||
|
Header names are merged case-insensitively. Custom headers override the default
|
||||||
|
`Content-Type`; a supplied `token` overrides custom `Authorization`, and HeaderSpec
|
||||||
|
query options override matching custom query headers. Without a token, custom
|
||||||
|
`Authorization` is preserved. Configuration is copied at construction; create or
|
||||||
|
retrieve a client with new configuration to change headers. Factory clients are
|
||||||
|
cached by URL and effective headers, keeping different tenants and tokens separate.
|
||||||
|
|
||||||
|
Grid adapters must forward `dataSourceOptions.headers` to this `headers` option.
|
||||||
+1
File diff suppressed because one or more lines are too long
+5
@@ -0,0 +1,5 @@
|
|||||||
|
export * from './common';
|
||||||
|
export * from './resolvespec';
|
||||||
|
export * from './websocketspec';
|
||||||
|
export * from './headerspec';
|
||||||
|
//# sourceMappingURL=index.d.ts.map
|
||||||
Vendored
+432
@@ -0,0 +1,432 @@
|
|||||||
|
import { v4 as e } from "uuid";
|
||||||
|
import { b64DecodeUnicode as t, b64EncodeUnicode as n } from "@warkypublic/artemis-kit/base64";
|
||||||
|
//#region src/common/http.ts
|
||||||
|
function r(...e) {
|
||||||
|
let t = {};
|
||||||
|
for (let n of e) for (let [e, r] of Object.entries(n)) {
|
||||||
|
for (let n of Object.keys(t)) n.toLowerCase() === e.toLowerCase() && delete t[n];
|
||||||
|
Object.defineProperty(t, e, {
|
||||||
|
value: r,
|
||||||
|
enumerable: !0,
|
||||||
|
configurable: !0,
|
||||||
|
writable: !0
|
||||||
|
});
|
||||||
|
}
|
||||||
|
return t;
|
||||||
|
}
|
||||||
|
function i(e) {
|
||||||
|
return r({ "Content-Type": "application/json" }, e.headers ?? {}, e.token ? { Authorization: `Bearer ${e.token}` } : {});
|
||||||
|
}
|
||||||
|
function a(e) {
|
||||||
|
let t = Object.entries(i(e)).map(([e, t]) => [e.toLowerCase(), t]).sort(([e], [t]) => e.localeCompare(t));
|
||||||
|
return JSON.stringify([e.baseUrl, t]);
|
||||||
|
}
|
||||||
|
//#endregion
|
||||||
|
//#region src/resolvespec/client.ts
|
||||||
|
var o = /* @__PURE__ */ new Map();
|
||||||
|
function s(e) {
|
||||||
|
let t = a(e), n = o.get(t);
|
||||||
|
return n || (n = new c(e), o.set(t, n)), n;
|
||||||
|
}
|
||||||
|
var c = class {
|
||||||
|
constructor(e) {
|
||||||
|
this.config = {
|
||||||
|
...e,
|
||||||
|
headers: { ...e.headers }
|
||||||
|
};
|
||||||
|
}
|
||||||
|
buildUrl(e, t, n) {
|
||||||
|
let r = `${this.config.baseUrl}/${e}/${t}`;
|
||||||
|
return n && (r += `/${n}`), r;
|
||||||
|
}
|
||||||
|
baseHeaders() {
|
||||||
|
return i(this.config);
|
||||||
|
}
|
||||||
|
async fetchWithError(e, t) {
|
||||||
|
let n = await fetch(e, t), r = await n.json();
|
||||||
|
if (!n.ok) throw Error(r.error?.message || "An error occurred");
|
||||||
|
return r;
|
||||||
|
}
|
||||||
|
async getMetadata(e, t) {
|
||||||
|
let n = this.buildUrl(e, t);
|
||||||
|
return this.fetchWithError(n, {
|
||||||
|
method: "GET",
|
||||||
|
headers: this.baseHeaders()
|
||||||
|
});
|
||||||
|
}
|
||||||
|
async read(e, t, n, r) {
|
||||||
|
let i = typeof n == "number" || typeof n == "string" ? String(n) : void 0, a = this.buildUrl(e, t, i), o = {
|
||||||
|
operation: "read",
|
||||||
|
id: Array.isArray(n) ? n : void 0,
|
||||||
|
options: r
|
||||||
|
};
|
||||||
|
return this.fetchWithError(a, {
|
||||||
|
method: "POST",
|
||||||
|
headers: this.baseHeaders(),
|
||||||
|
body: JSON.stringify(o)
|
||||||
|
});
|
||||||
|
}
|
||||||
|
async create(e, t, n, r) {
|
||||||
|
let i = this.buildUrl(e, t), a = {
|
||||||
|
operation: "create",
|
||||||
|
data: n,
|
||||||
|
options: r
|
||||||
|
};
|
||||||
|
return this.fetchWithError(i, {
|
||||||
|
method: "POST",
|
||||||
|
headers: this.baseHeaders(),
|
||||||
|
body: JSON.stringify(a)
|
||||||
|
});
|
||||||
|
}
|
||||||
|
async update(e, t, n, r, i) {
|
||||||
|
let a = typeof r == "number" || typeof r == "string" ? String(r) : void 0, o = this.buildUrl(e, t, a), s = {
|
||||||
|
operation: "update",
|
||||||
|
id: Array.isArray(r) ? r : void 0,
|
||||||
|
data: n,
|
||||||
|
options: i
|
||||||
|
};
|
||||||
|
return this.fetchWithError(o, {
|
||||||
|
method: "POST",
|
||||||
|
headers: this.baseHeaders(),
|
||||||
|
body: JSON.stringify(s)
|
||||||
|
});
|
||||||
|
}
|
||||||
|
async delete(e, t, n) {
|
||||||
|
let r = this.buildUrl(e, t, String(n));
|
||||||
|
return this.fetchWithError(r, {
|
||||||
|
method: "POST",
|
||||||
|
headers: this.baseHeaders(),
|
||||||
|
body: JSON.stringify({ operation: "delete" })
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}, l = /* @__PURE__ */ new Map();
|
||||||
|
function u(e) {
|
||||||
|
let t = e.url, n = l.get(t);
|
||||||
|
return n || (n = new d(e), l.set(t, n)), n;
|
||||||
|
}
|
||||||
|
var d = class {
|
||||||
|
constructor(e) {
|
||||||
|
this.ws = null, this.messageHandlers = /* @__PURE__ */ new Map(), this.subscriptions = /* @__PURE__ */ new Map(), this.eventListeners = {}, this.state = "disconnected", this.reconnectAttempts = 0, this.reconnectTimer = null, this.heartbeatTimer = null, this.isManualClose = !1, this.config = {
|
||||||
|
url: e.url,
|
||||||
|
reconnect: e.reconnect ?? !0,
|
||||||
|
reconnectInterval: e.reconnectInterval ?? 3e3,
|
||||||
|
maxReconnectAttempts: e.maxReconnectAttempts ?? 10,
|
||||||
|
heartbeatInterval: e.heartbeatInterval ?? 3e4,
|
||||||
|
debug: e.debug ?? !1
|
||||||
|
};
|
||||||
|
}
|
||||||
|
async connect() {
|
||||||
|
if (this.ws?.readyState === WebSocket.OPEN) {
|
||||||
|
this.log("Already connected");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
return this.isManualClose = !1, this.setState("connecting"), new Promise((e, t) => {
|
||||||
|
try {
|
||||||
|
this.ws = new WebSocket(this.config.url), this.ws.onopen = () => {
|
||||||
|
this.log("Connected to WebSocket server"), this.setState("connected"), this.reconnectAttempts = 0, this.startHeartbeat(), this.emit("connect"), e();
|
||||||
|
}, this.ws.onmessage = (e) => {
|
||||||
|
this.handleMessage(e.data);
|
||||||
|
}, this.ws.onerror = (e) => {
|
||||||
|
this.log("WebSocket error:", e);
|
||||||
|
let n = /* @__PURE__ */ Error("WebSocket connection error");
|
||||||
|
this.emit("error", n), t(n);
|
||||||
|
}, this.ws.onclose = (e) => {
|
||||||
|
this.log("WebSocket closed:", e.code, e.reason), this.stopHeartbeat(), this.setState("disconnected"), this.emit("disconnect", e), this.config.reconnect && !this.isManualClose && this.reconnectAttempts < this.config.maxReconnectAttempts && (this.reconnectAttempts++, this.log(`Reconnection attempt ${this.reconnectAttempts}/${this.config.maxReconnectAttempts}`), this.setState("reconnecting"), this.reconnectTimer = setTimeout(() => {
|
||||||
|
this.connect().catch((e) => {
|
||||||
|
this.log("Reconnection failed:", e);
|
||||||
|
});
|
||||||
|
}, this.config.reconnectInterval));
|
||||||
|
};
|
||||||
|
} catch (e) {
|
||||||
|
t(e);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
disconnect() {
|
||||||
|
this.isManualClose = !0, this.reconnectTimer &&= (clearTimeout(this.reconnectTimer), null), this.stopHeartbeat(), this.ws &&= (this.setState("disconnecting"), this.ws.close(), null), this.setState("disconnected"), this.messageHandlers.clear();
|
||||||
|
}
|
||||||
|
async request(t, n, r) {
|
||||||
|
this.ensureConnected();
|
||||||
|
let i = e(), a = {
|
||||||
|
id: i,
|
||||||
|
type: "request",
|
||||||
|
operation: t,
|
||||||
|
entity: n,
|
||||||
|
schema: r?.schema,
|
||||||
|
record_id: r?.record_id,
|
||||||
|
data: r?.data,
|
||||||
|
options: r?.options
|
||||||
|
};
|
||||||
|
return new Promise((e, t) => {
|
||||||
|
this.messageHandlers.set(i, (n) => {
|
||||||
|
n.success ? e(n.data) : t(Error(n.error?.message || "Request failed"));
|
||||||
|
}), this.send(a), setTimeout(() => {
|
||||||
|
this.messageHandlers.has(i) && (this.messageHandlers.delete(i), t(/* @__PURE__ */ Error("Request timeout")));
|
||||||
|
}, 3e4);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
async read(e, t) {
|
||||||
|
return this.request("read", e, {
|
||||||
|
schema: t?.schema,
|
||||||
|
record_id: t?.record_id,
|
||||||
|
options: {
|
||||||
|
filters: t?.filters,
|
||||||
|
columns: t?.columns,
|
||||||
|
sort: t?.sort,
|
||||||
|
preload: t?.preload,
|
||||||
|
limit: t?.limit,
|
||||||
|
offset: t?.offset
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
async create(e, t, n) {
|
||||||
|
return this.request("create", e, {
|
||||||
|
schema: n?.schema,
|
||||||
|
data: t
|
||||||
|
});
|
||||||
|
}
|
||||||
|
async update(e, t, n, r) {
|
||||||
|
return this.request("update", e, {
|
||||||
|
schema: r?.schema,
|
||||||
|
record_id: t,
|
||||||
|
data: n
|
||||||
|
});
|
||||||
|
}
|
||||||
|
async delete(e, t, n) {
|
||||||
|
await this.request("delete", e, {
|
||||||
|
schema: n?.schema,
|
||||||
|
record_id: t
|
||||||
|
});
|
||||||
|
}
|
||||||
|
async meta(e, t) {
|
||||||
|
return this.request("meta", e, { schema: t?.schema });
|
||||||
|
}
|
||||||
|
async subscribe(t, n, r) {
|
||||||
|
this.ensureConnected();
|
||||||
|
let i = e(), a = {
|
||||||
|
id: i,
|
||||||
|
type: "subscription",
|
||||||
|
operation: "subscribe",
|
||||||
|
entity: t,
|
||||||
|
schema: r?.schema,
|
||||||
|
options: { filters: r?.filters }
|
||||||
|
};
|
||||||
|
return new Promise((e, o) => {
|
||||||
|
this.messageHandlers.set(i, (i) => {
|
||||||
|
if (i.success && i.data?.subscription_id) {
|
||||||
|
let a = i.data.subscription_id;
|
||||||
|
this.subscriptions.set(a, {
|
||||||
|
id: a,
|
||||||
|
entity: t,
|
||||||
|
schema: r?.schema,
|
||||||
|
options: { filters: r?.filters },
|
||||||
|
callback: n
|
||||||
|
}), this.log(`Subscribed to ${t} with ID: ${a}`), e(a);
|
||||||
|
} else o(Error(i.error?.message || "Subscription failed"));
|
||||||
|
}), this.send(a), setTimeout(() => {
|
||||||
|
this.messageHandlers.has(i) && (this.messageHandlers.delete(i), o(/* @__PURE__ */ Error("Subscription timeout")));
|
||||||
|
}, 1e4);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
async unsubscribe(t) {
|
||||||
|
this.ensureConnected();
|
||||||
|
let n = e(), r = {
|
||||||
|
id: n,
|
||||||
|
type: "subscription",
|
||||||
|
operation: "unsubscribe",
|
||||||
|
subscription_id: t
|
||||||
|
};
|
||||||
|
return new Promise((e, i) => {
|
||||||
|
this.messageHandlers.set(n, (n) => {
|
||||||
|
n.success ? (this.subscriptions.delete(t), this.log(`Unsubscribed from ${t}`), e()) : i(Error(n.error?.message || "Unsubscribe failed"));
|
||||||
|
}), this.send(r), setTimeout(() => {
|
||||||
|
this.messageHandlers.has(n) && (this.messageHandlers.delete(n), i(/* @__PURE__ */ Error("Unsubscribe timeout")));
|
||||||
|
}, 1e4);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
getSubscriptions() {
|
||||||
|
return Array.from(this.subscriptions.values());
|
||||||
|
}
|
||||||
|
getState() {
|
||||||
|
return this.state;
|
||||||
|
}
|
||||||
|
isConnected() {
|
||||||
|
return this.ws?.readyState === WebSocket.OPEN;
|
||||||
|
}
|
||||||
|
on(e, t) {
|
||||||
|
this.eventListeners[e] = t;
|
||||||
|
}
|
||||||
|
off(e) {
|
||||||
|
delete this.eventListeners[e];
|
||||||
|
}
|
||||||
|
handleMessage(e) {
|
||||||
|
try {
|
||||||
|
let t = JSON.parse(e);
|
||||||
|
switch (this.log("Received message:", t), this.emit("message", t), t.type) {
|
||||||
|
case "response":
|
||||||
|
this.handleResponse(t);
|
||||||
|
break;
|
||||||
|
case "notification":
|
||||||
|
this.handleNotification(t);
|
||||||
|
break;
|
||||||
|
case "pong": break;
|
||||||
|
default: this.log("Unknown message type:", t.type);
|
||||||
|
}
|
||||||
|
} catch (e) {
|
||||||
|
this.log("Error parsing message:", e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
handleResponse(e) {
|
||||||
|
let t = this.messageHandlers.get(e.id);
|
||||||
|
t && (t(e), this.messageHandlers.delete(e.id));
|
||||||
|
}
|
||||||
|
handleNotification(e) {
|
||||||
|
let t = this.subscriptions.get(e.subscription_id);
|
||||||
|
t?.callback && t.callback(e);
|
||||||
|
}
|
||||||
|
send(e) {
|
||||||
|
if (!this.ws || this.ws.readyState !== WebSocket.OPEN) throw Error("WebSocket is not connected");
|
||||||
|
let t = JSON.stringify(e);
|
||||||
|
this.log("Sending message:", e), this.ws.send(t);
|
||||||
|
}
|
||||||
|
startHeartbeat() {
|
||||||
|
this.heartbeatTimer ||= setInterval(() => {
|
||||||
|
if (this.isConnected()) {
|
||||||
|
let t = {
|
||||||
|
id: e(),
|
||||||
|
type: "ping"
|
||||||
|
};
|
||||||
|
this.send(t);
|
||||||
|
}
|
||||||
|
}, this.config.heartbeatInterval);
|
||||||
|
}
|
||||||
|
stopHeartbeat() {
|
||||||
|
this.heartbeatTimer &&= (clearInterval(this.heartbeatTimer), null);
|
||||||
|
}
|
||||||
|
setState(e) {
|
||||||
|
this.state !== e && (this.state = e, this.emit("stateChange", e));
|
||||||
|
}
|
||||||
|
ensureConnected() {
|
||||||
|
if (!this.isConnected()) throw Error("WebSocket is not connected. Call connect() first.");
|
||||||
|
}
|
||||||
|
emit(e, ...t) {
|
||||||
|
let n = this.eventListeners[e];
|
||||||
|
n && n(...t);
|
||||||
|
}
|
||||||
|
log(...e) {
|
||||||
|
this.config.debug && console.log("[WebSocketClient]", ...e);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
//#endregion
|
||||||
|
//#region src/headerspec/client.ts
|
||||||
|
function f(e) {
|
||||||
|
return "ZIP_" + n(e);
|
||||||
|
}
|
||||||
|
function p(e) {
|
||||||
|
let t = e;
|
||||||
|
return t.startsWith("ZIP_") ? (t = t.slice(4).replace(/[\n\r ]/g, ""), t = m(t)) : t.startsWith("__") && (t = t.slice(2).replace(/[\n\r ]/g, ""), t = m(t)), (t.startsWith("ZIP_") || t.startsWith("__")) && (t = p(t)), t;
|
||||||
|
}
|
||||||
|
function m(e) {
|
||||||
|
return t(e);
|
||||||
|
}
|
||||||
|
function h(e) {
|
||||||
|
let t = {};
|
||||||
|
if (e.columns?.length && (t["X-Select-Fields"] = e.columns.join(",")), e.omit_columns?.length && (t["X-Not-Select-Fields"] = e.omit_columns.join(",")), e.filters?.length) for (let n of e.filters) {
|
||||||
|
let e = n.logic_operator ?? "AND", r = g(n.operator), i = _(n);
|
||||||
|
n.operator === "eq" && e === "AND" ? t[`X-FieldFilter-${n.column}`] = i : e === "OR" ? t[`X-SearchOr-${r}-${n.column}`] = i : t[`X-SearchOp-${r}-${n.column}`] = i;
|
||||||
|
}
|
||||||
|
if (e.sort?.length && (t["X-Sort"] = e.sort.map((e) => e.direction.toUpperCase() === "DESC" ? `-${e.column}` : `+${e.column}`).join(",")), e.limit !== void 0 && (t["X-Limit"] = String(e.limit)), e.offset !== void 0 && (t["X-Offset"] = String(e.offset)), e.cursor_forward && (t["X-Cursor-Forward"] = e.cursor_forward), e.cursor_backward && (t["X-Cursor-Backward"] = e.cursor_backward), e.preload?.length && (t["X-Preload"] = e.preload.map((e) => e.columns?.length ? `${e.relation}:${e.columns.join(",")}` : e.relation).join("|")), e.fetch_row_number && (t["X-Fetch-RowNumber"] = e.fetch_row_number), e.computedColumns?.length) for (let n of e.computedColumns) t[`X-CQL-SEL-${n.name}`] = n.expression;
|
||||||
|
return e.customOperators?.length && (t["X-Custom-SQL-W"] = e.customOperators.map((e) => e.sql).join(" AND ")), t;
|
||||||
|
}
|
||||||
|
function g(e) {
|
||||||
|
switch (e) {
|
||||||
|
case "eq": return "equals";
|
||||||
|
case "neq": return "notequals";
|
||||||
|
case "gt": return "greaterthan";
|
||||||
|
case "gte": return "greaterthanorequal";
|
||||||
|
case "lt": return "lessthan";
|
||||||
|
case "lte": return "lessthanorequal";
|
||||||
|
case "like":
|
||||||
|
case "ilike":
|
||||||
|
case "contains": return "contains";
|
||||||
|
case "startswith": return "beginswith";
|
||||||
|
case "endswith": return "endswith";
|
||||||
|
case "in": return "in";
|
||||||
|
case "between": return "between";
|
||||||
|
case "between_inclusive": return "betweeninclusive";
|
||||||
|
case "is_null": return "empty";
|
||||||
|
case "is_not_null": return "notempty";
|
||||||
|
default: return e;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
function _(e) {
|
||||||
|
return e.value === null || e.value === void 0 ? "" : Array.isArray(e.value) ? e.value.join(",") : String(e.value);
|
||||||
|
}
|
||||||
|
var v = /* @__PURE__ */ new Map();
|
||||||
|
function y(e) {
|
||||||
|
let t = a(e), n = v.get(t);
|
||||||
|
return n || (n = new b(e), v.set(t, n)), n;
|
||||||
|
}
|
||||||
|
var b = class {
|
||||||
|
constructor(e) {
|
||||||
|
this.config = {
|
||||||
|
...e,
|
||||||
|
headers: { ...e.headers }
|
||||||
|
};
|
||||||
|
}
|
||||||
|
buildUrl(e, t, n) {
|
||||||
|
let r = `${this.config.baseUrl}/${e}/${t}`;
|
||||||
|
return n && (r += `/${n}`), r;
|
||||||
|
}
|
||||||
|
baseHeaders() {
|
||||||
|
return i(this.config);
|
||||||
|
}
|
||||||
|
async fetchWithError(e, t) {
|
||||||
|
let n = await fetch(e, t), r = await n.json();
|
||||||
|
if (!n.ok) throw Error(r.error?.message || `${n.statusText} (${n.status})`);
|
||||||
|
return {
|
||||||
|
data: r,
|
||||||
|
success: !0,
|
||||||
|
error: r.error ? r.error : void 0,
|
||||||
|
metadata: {
|
||||||
|
count: n.headers.get("content-range") ? Number(n.headers.get("content-range")?.split("/")[1]) : 0,
|
||||||
|
total: n.headers.get("content-range") ? Number(n.headers.get("content-range")?.split("/")[1]) : 0,
|
||||||
|
filtered: n.headers.get("content-range") ? Number(n.headers.get("content-range")?.split("/")[1]) : 0,
|
||||||
|
offset: n.headers.get("content-range") ? Number(n.headers.get("content-range")?.split("/")[0].split("-")[0]) : 0,
|
||||||
|
limit: n.headers.get("x-limit") ? Number(n.headers.get("x-limit")) : 0
|
||||||
|
}
|
||||||
|
};
|
||||||
|
}
|
||||||
|
async read(e, t, n, i) {
|
||||||
|
let a = this.buildUrl(e, t, n), o = i ? h(i) : {};
|
||||||
|
return this.fetchWithError(a, {
|
||||||
|
method: "GET",
|
||||||
|
headers: r(this.baseHeaders(), o)
|
||||||
|
});
|
||||||
|
}
|
||||||
|
async create(e, t, n, i) {
|
||||||
|
let a = this.buildUrl(e, t), o = i ? h(i) : {};
|
||||||
|
return this.fetchWithError(a, {
|
||||||
|
method: "POST",
|
||||||
|
headers: r(this.baseHeaders(), o),
|
||||||
|
body: JSON.stringify(n)
|
||||||
|
});
|
||||||
|
}
|
||||||
|
async update(e, t, n, i, a) {
|
||||||
|
let o = this.buildUrl(e, t, n), s = a ? h(a) : {};
|
||||||
|
return this.fetchWithError(o, {
|
||||||
|
method: "PUT",
|
||||||
|
headers: r(this.baseHeaders(), s),
|
||||||
|
body: JSON.stringify(i)
|
||||||
|
});
|
||||||
|
}
|
||||||
|
async delete(e, t, n) {
|
||||||
|
let r = this.buildUrl(e, t, n);
|
||||||
|
return this.fetchWithError(r, {
|
||||||
|
method: "DELETE",
|
||||||
|
headers: this.baseHeaders()
|
||||||
|
});
|
||||||
|
}
|
||||||
|
};
|
||||||
|
//#endregion
|
||||||
|
export { b as HeaderSpecClient, c as ResolveSpecClient, d as WebSocketClient, h as buildHeaders, p as decodeHeaderValue, f as encodeHeaderValue, y as getHeaderSpecClient, s as getResolveSpecClient, u as getWebSocketClient };
|
||||||
@@ -1,6 +1,6 @@
|
|||||||
{
|
{
|
||||||
"name": "@warkypublic/resolvespec-js",
|
"name": "@warkypublic/resolvespec-js",
|
||||||
"version": "1.0.1",
|
"version": "1.0.2",
|
||||||
"description": "TypeScript client library for ResolveSpec REST, HeaderSpec, and WebSocket APIs",
|
"description": "TypeScript client library for ResolveSpec REST, HeaderSpec, and WebSocket APIs",
|
||||||
"type": "module",
|
"type": "module",
|
||||||
"main": "./dist/index.cjs",
|
"main": "./dist/index.cjs",
|
||||||
@@ -38,20 +38,22 @@
|
|||||||
"author": "Hein (Warkanum) Puth",
|
"author": "Hein (Warkanum) Puth",
|
||||||
"license": "MIT",
|
"license": "MIT",
|
||||||
"dependencies": {
|
"dependencies": {
|
||||||
"uuid": "^13.0.0"
|
"@warkypublic/artemis-kit": "^1.0.10",
|
||||||
|
"uuid": "^14.0.2"
|
||||||
},
|
},
|
||||||
"devDependencies": {
|
"devDependencies": {
|
||||||
"@changesets/cli": "^2.29.8",
|
"@changesets/cli": "^3.0.3",
|
||||||
"@eslint/js": "^10.0.1",
|
"@eslint/js": "^10.0.1",
|
||||||
"@types/jsdom": "^27.0.0",
|
"@types/jsdom": "^30.0.0",
|
||||||
"eslint": "^10.0.0",
|
"@types/node": "^26.6.2",
|
||||||
"globals": "^17.3.0",
|
"eslint": "^10.11.0",
|
||||||
"jsdom": "^28.1.0",
|
"globals": "^17.12.0",
|
||||||
"typescript": "^5.9.3",
|
"jsdom": "^30.1.1",
|
||||||
"typescript-eslint": "^8.55.0",
|
"typescript": "^6.0.3",
|
||||||
"vite": "^7.3.1",
|
"typescript-eslint": "^8.70.1",
|
||||||
"vite-plugin-dts": "^4.5.4",
|
"vite": "^8.3.0",
|
||||||
"vitest": "^4.0.18"
|
"vite-plugin-dts": "^5.1.1",
|
||||||
|
"vitest": "^5.0.1"
|
||||||
},
|
},
|
||||||
"engines": {
|
"engines": {
|
||||||
"node": ">=18"
|
"node": ">=18"
|
||||||
+1283
-1293
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,5 @@
|
|||||||
|
packages:
|
||||||
|
- '.'
|
||||||
|
|
||||||
|
allowBuilds:
|
||||||
|
esbuild: true
|
||||||
@@ -0,0 +1,65 @@
|
|||||||
|
import { afterEach, describe, expect, it, vi } from 'vitest';
|
||||||
|
import { ResolveSpecClient, getResolveSpecClient } from '../resolvespec/client';
|
||||||
|
import { HeaderSpecClient, getHeaderSpecClient } from '../headerspec/client';
|
||||||
|
|
||||||
|
afterEach(() => vi.unstubAllGlobals());
|
||||||
|
|
||||||
|
for (const [name, Client, factory] of [
|
||||||
|
['ResolveSpec', ResolveSpecClient, getResolveSpecClient],
|
||||||
|
['HeaderSpec', HeaderSpecClient, getHeaderSpecClient],
|
||||||
|
] as const) {
|
||||||
|
describe(`${name} custom headers`, () => {
|
||||||
|
it('sends tenant headers on every operation and resolves collisions case-insensitively', async () => {
|
||||||
|
const fetchMock = vi.fn().mockResolvedValue({
|
||||||
|
ok: true, headers: new Headers(), json: async () => ({ success: true, data: [] }),
|
||||||
|
});
|
||||||
|
vi.stubGlobal('fetch', fetchMock);
|
||||||
|
const headers = { 'X-Tenant': 'acme', authorization: 'Basic ignored', 'content-type': 'application/custom+json', 'x-limit': '99' };
|
||||||
|
const client = new Client({ baseUrl: 'http://localhost:3000', token: 'tok', headers });
|
||||||
|
await client.read('public', 'users', undefined, { limit: 10 });
|
||||||
|
await client.create('public', 'users', {});
|
||||||
|
if (client instanceof ResolveSpecClient) {
|
||||||
|
await client.update('public', 'users', {}, '1');
|
||||||
|
await client.getMetadata('public', 'users');
|
||||||
|
} else {
|
||||||
|
await client.update('public', 'users', '1', {});
|
||||||
|
}
|
||||||
|
await client.delete('public', 'users', '1');
|
||||||
|
for (const [, init] of fetchMock.mock.calls) {
|
||||||
|
const sent = new Headers(init.headers);
|
||||||
|
expect(sent.get('x-tenant')).toBe('acme');
|
||||||
|
expect(sent.get('authorization')).toBe('Bearer tok');
|
||||||
|
expect(sent.get('content-type')).toBe('application/custom+json');
|
||||||
|
}
|
||||||
|
if (client instanceof HeaderSpecClient) {
|
||||||
|
expect(new Headers(fetchMock.mock.calls[0][1].headers).get('x-limit')).toBe('10');
|
||||||
|
}
|
||||||
|
expect(headers.authorization).toBe('Basic ignored');
|
||||||
|
expect(headers['x-limit']).toBe('99');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('supports custom authentication without a token', async () => {
|
||||||
|
const fetchMock = vi.fn().mockResolvedValue({
|
||||||
|
ok: true, headers: new Headers(), json: async () => ({ success: true, data: [] }),
|
||||||
|
});
|
||||||
|
vi.stubGlobal('fetch', fetchMock);
|
||||||
|
await new Client({ baseUrl: 'http://localhost:3000', headers: { Authorization: 'Basic custom' } }).read('public', 'users');
|
||||||
|
expect(new Headers(fetchMock.mock.calls[0][1].headers).get('authorization')).toBe('Basic custom');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('isolates cached clients by headers and token, and snapshots configuration', async () => {
|
||||||
|
const config = { baseUrl: 'http://tenant-cache', token: 'one', headers: { 'X-Tenant': 'acme', 'X-App': 'grid' } };
|
||||||
|
const first = factory(config);
|
||||||
|
expect(factory({ ...config, headers: { 'x-app': 'grid', 'x-tenant': 'acme' } })).toBe(first);
|
||||||
|
expect(factory({ ...config, token: 'two' })).not.toBe(first);
|
||||||
|
config.headers['X-Tenant'] = 'other';
|
||||||
|
expect(factory(config)).not.toBe(first);
|
||||||
|
const fetchMock = vi.fn().mockResolvedValue({
|
||||||
|
ok: true, headers: new Headers(), json: async () => ({ success: true, data: [] }),
|
||||||
|
});
|
||||||
|
vi.stubGlobal('fetch', fetchMock);
|
||||||
|
await first.read('public', 'users');
|
||||||
|
expect(new Headers(fetchMock.mock.calls[0][1].headers).get('x-tenant')).toBe('acme');
|
||||||
|
});
|
||||||
|
});
|
||||||
|
}
|
||||||
+107
@@ -2,6 +2,101 @@ import { describe, it, expect, vi, beforeEach } from 'vitest';
|
|||||||
import { buildHeaders, encodeHeaderValue, decodeHeaderValue, HeaderSpecClient, getHeaderSpecClient } from '../headerspec/client';
|
import { buildHeaders, encodeHeaderValue, decodeHeaderValue, HeaderSpecClient, getHeaderSpecClient } from '../headerspec/client';
|
||||||
import type { Options, ClientConfig, APIResponse } from '../common/types';
|
import type { Options, ClientConfig, APIResponse } from '../common/types';
|
||||||
|
|
||||||
|
describe('buildHeaders (extended restheadspec options)', () => {
|
||||||
|
it('should set X-Preload-Where when all preloads share one where', () => {
|
||||||
|
const h = buildHeaders({
|
||||||
|
preload: [
|
||||||
|
{ relation: 'Items', columns: ['id'], where: 'active = true' },
|
||||||
|
{ relation: 'Tags', where: 'active = true' },
|
||||||
|
],
|
||||||
|
});
|
||||||
|
expect(h['X-Preload']).toBe('Items:id|Tags');
|
||||||
|
expect(h['X-Preload-Where']).toBe('active = true');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should use numbered headers for mixed where clauses', () => {
|
||||||
|
const h = buildHeaders({
|
||||||
|
preload: [
|
||||||
|
{ relation: 'Items', where: 'a = 1' },
|
||||||
|
{ relation: 'Category' },
|
||||||
|
{ relation: 'Tags', where: 'b = 2' },
|
||||||
|
],
|
||||||
|
});
|
||||||
|
expect(h['X-Preload']).toBe('Category');
|
||||||
|
expect(h['X-Preload-Where']).toBeUndefined();
|
||||||
|
expect(h['X-Preload-1']).toBe('Items');
|
||||||
|
expect(h['X-Preload-1-Where']).toBe('a = 1');
|
||||||
|
expect(h['X-Preload-2']).toBe('Tags');
|
||||||
|
expect(h['X-Preload-2-Where']).toBe('b = 2');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should set expand, joins, or-sql, search cols, advsql', () => {
|
||||||
|
const h = buildHeaders({
|
||||||
|
expand: [{ relation: 'Dept', columns: ['id', 'name'] }, { relation: 'Role' }],
|
||||||
|
custom_sql_joins: ['LEFT JOIN a ON a.id = b.id', 'INNER JOIN c ON c.id = b.cid'],
|
||||||
|
custom_sql_or: ['x = 1', 'y = 2'],
|
||||||
|
search_columns: ['name', 'email'],
|
||||||
|
advanced_sql: { total: 'a + b' },
|
||||||
|
});
|
||||||
|
expect(h['X-Expand']).toBe('Dept:id,name|Role');
|
||||||
|
expect(h['X-Custom-SQL-Join']).toBe('LEFT JOIN a ON a.id = b.id|INNER JOIN c ON c.id = b.cid');
|
||||||
|
expect(h['X-Custom-SQL-Or']).toBe('x = 1 OR y = 2');
|
||||||
|
expect(h['X-SearchCols']).toBe('name,email');
|
||||||
|
expect(h['X-AdvSQL-total']).toBe('a + b');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should set boolean flags, pk row and response format', () => {
|
||||||
|
const h = buildHeaders({
|
||||||
|
clean_json: true,
|
||||||
|
distinct: true,
|
||||||
|
skip_count: true,
|
||||||
|
skip_cache: false,
|
||||||
|
atomic_transaction: true,
|
||||||
|
single_record_as_object: false,
|
||||||
|
pk_row: '42',
|
||||||
|
response_format: 'detail',
|
||||||
|
});
|
||||||
|
expect(h['X-Clean-JSON']).toBe('true');
|
||||||
|
expect(h['X-Distinct']).toBe('true');
|
||||||
|
expect(h['X-SkipCount']).toBe('true');
|
||||||
|
expect(h['X-SkipCache']).toBe('false');
|
||||||
|
expect(h['X-Transaction-Atomic']).toBe('true');
|
||||||
|
expect(h['X-Single-Record-As-Object']).toBe('false');
|
||||||
|
expect(h['X-PKRow']).toBe('42');
|
||||||
|
expect(h['X-DetailApi']).toBe('true');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should set spatial and vector filters as JSON', () => {
|
||||||
|
const h = buildHeaders({
|
||||||
|
filters: [
|
||||||
|
{ column: 'geom', operator: 'st_dwithin', value: { geom: 'POINT(0 0)', distance: 5 }, logic_operator: 'OR' },
|
||||||
|
{ column: 'emb', operator: 'cosine_within', value: { vector: [1, 2], distance: 0.3 } },
|
||||||
|
],
|
||||||
|
});
|
||||||
|
expect(JSON.parse(h['X-SpatialFilter-geom'])).toEqual({
|
||||||
|
op: 'st_dwithin', value: { geom: 'POINT(0 0)', distance: 5 }, logic: 'or',
|
||||||
|
});
|
||||||
|
expect(JSON.parse(h['X-VectorFilter-emb']).op).toBe('cosine_within');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should set vector search headers', () => {
|
||||||
|
const h = buildHeaders({
|
||||||
|
vector_search: { column: 'emb', vector: [0.1, 0.2], metric: 'cosine', as: 'dist', direction: 'desc' },
|
||||||
|
});
|
||||||
|
expect(h['X-Vector-Search-emb']).toBe('cosine');
|
||||||
|
expect(h['X-Vector-Search-Vector']).toBe('[0.1,0.2]');
|
||||||
|
expect(h['X-Vector-Search-As']).toBe('dist');
|
||||||
|
expect(h['X-Vector-Search-Dir']).toBe('desc');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should encode X-Files as ZIP_ JSON', () => {
|
||||||
|
const xf = { tablename: 'users', prefix: 'USR', limit: 10 };
|
||||||
|
const h = buildHeaders({ xfiles: xf });
|
||||||
|
expect(h['X-Files'].startsWith('ZIP_')).toBe(true);
|
||||||
|
expect(JSON.parse(decodeHeaderValue(h['X-Files']))).toEqual(xf);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
describe('buildHeaders', () => {
|
describe('buildHeaders', () => {
|
||||||
it('should set X-Select-Fields for columns', () => {
|
it('should set X-Select-Fields for columns', () => {
|
||||||
const h = buildHeaders({ columns: ['id', 'name', 'email'] });
|
const h = buildHeaders({ columns: ['id', 'name', 'email'] });
|
||||||
@@ -126,11 +221,22 @@ describe('encodeHeaderValue / decodeHeaderValue', () => {
|
|||||||
expect(decoded).toBe(original);
|
expect(decoded).toBe(original);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it('should round-trip UTF-8 values', () => {
|
||||||
|
const original = 'café ☕ 你好';
|
||||||
|
expect(decodeHeaderValue(encodeHeaderValue(original))).toBe(original);
|
||||||
|
});
|
||||||
|
|
||||||
it('should decode __ prefixed values', () => {
|
it('should decode __ prefixed values', () => {
|
||||||
const encoded = '__' + btoa('hello');
|
const encoded = '__' + btoa('hello');
|
||||||
expect(decodeHeaderValue(encoded)).toBe('hello');
|
expect(decodeHeaderValue(encoded)).toBe('hello');
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it('should decode UTF-8 values with the __ prefix', () => {
|
||||||
|
const bytes = new TextEncoder().encode('café ☕');
|
||||||
|
const binary = Array.from(bytes, (byte) => String.fromCharCode(byte)).join('');
|
||||||
|
expect(decodeHeaderValue('__' + btoa(binary))).toBe('café ☕');
|
||||||
|
});
|
||||||
|
|
||||||
it('should return plain values as-is', () => {
|
it('should return plain values as-is', () => {
|
||||||
expect(decodeHeaderValue('plain')).toBe('plain');
|
expect(decodeHeaderValue('plain')).toBe('plain');
|
||||||
});
|
});
|
||||||
@@ -142,6 +248,7 @@ describe('HeaderSpecClient', () => {
|
|||||||
function mockFetch<T>(data: APIResponse<T>, ok = true) {
|
function mockFetch<T>(data: APIResponse<T>, ok = true) {
|
||||||
return vi.fn().mockResolvedValue({
|
return vi.fn().mockResolvedValue({
|
||||||
ok,
|
ok,
|
||||||
|
headers: new Headers(),
|
||||||
json: () => Promise.resolve(data),
|
json: () => Promise.resolve(data),
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
@@ -0,0 +1,30 @@
|
|||||||
|
import type { ClientConfig } from './types';
|
||||||
|
|
||||||
|
/** Merge HTTP headers case-insensitively, preserving the winning spelling. */
|
||||||
|
export function mergeHeaders(...sources: Record<string, string>[]): Record<string, string> {
|
||||||
|
const result: Record<string, string> = {};
|
||||||
|
for (const source of sources) {
|
||||||
|
for (const [name, value] of Object.entries(source)) {
|
||||||
|
for (const existing of Object.keys(result)) {
|
||||||
|
if (existing.toLowerCase() === name.toLowerCase()) delete result[existing];
|
||||||
|
}
|
||||||
|
Object.defineProperty(result, name, { value, enumerable: true, configurable: true, writable: true });
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function clientHeaders(config: ClientConfig): Record<string, string> {
|
||||||
|
return mergeHeaders(
|
||||||
|
{ 'Content-Type': 'application/json' },
|
||||||
|
config.headers ?? {},
|
||||||
|
config.token ? { Authorization: `Bearer ${config.token}` } : {},
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function clientCacheKey(config: ClientConfig): string {
|
||||||
|
const headers = Object.entries(clientHeaders(config))
|
||||||
|
.map(([name, value]) => [name.toLowerCase(), value])
|
||||||
|
.sort(([a], [b]) => a.localeCompare(b));
|
||||||
|
return JSON.stringify([config.baseUrl, headers]);
|
||||||
|
}
|
||||||
@@ -0,0 +1,216 @@
|
|||||||
|
// Types aligned with Go pkg/common/types.go
|
||||||
|
|
||||||
|
export type Operator =
|
||||||
|
| 'eq' | 'neq' | 'gt' | 'gte' | 'lt' | 'lte'
|
||||||
|
| 'like' | 'ilike' | 'in'
|
||||||
|
| 'contains' | 'startswith' | 'endswith'
|
||||||
|
| 'between' | 'between_inclusive'
|
||||||
|
| 'is_null' | 'is_not_null'
|
||||||
|
// PostGIS spatial (sent via X-SpatialFilter-{col})
|
||||||
|
| 'st_dwithin' | 'bbox'
|
||||||
|
// pgvector similarity (sent via X-VectorFilter-{col})
|
||||||
|
| 'l2_within' | 'cosine_within' | 'ip_within';
|
||||||
|
|
||||||
|
export type Operation = 'read' | 'create' | 'update' | 'delete';
|
||||||
|
export type SortDirection = 'asc' | 'desc' | 'ASC' | 'DESC';
|
||||||
|
|
||||||
|
export interface Parameter {
|
||||||
|
name: string;
|
||||||
|
value: string;
|
||||||
|
sequence?: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface PreloadOption {
|
||||||
|
relation: string;
|
||||||
|
table_name?: string;
|
||||||
|
columns?: string[];
|
||||||
|
omit_columns?: string[];
|
||||||
|
sort?: SortOption[];
|
||||||
|
filters?: FilterOption[];
|
||||||
|
where?: string;
|
||||||
|
limit?: number;
|
||||||
|
offset?: number;
|
||||||
|
updatable?: boolean;
|
||||||
|
computed_ql?: Record<string, string>;
|
||||||
|
recursive?: boolean;
|
||||||
|
// Relationship keys
|
||||||
|
primary_key?: string;
|
||||||
|
related_key?: string;
|
||||||
|
foreign_key?: string;
|
||||||
|
recursive_child_key?: string;
|
||||||
|
// Custom SQL JOINs
|
||||||
|
sql_joins?: string[];
|
||||||
|
join_aliases?: string[];
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface FilterOption {
|
||||||
|
column: string;
|
||||||
|
operator: Operator | string;
|
||||||
|
value: any;
|
||||||
|
logic_operator?: 'AND' | 'OR';
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface SortOption {
|
||||||
|
column: string;
|
||||||
|
direction: SortDirection;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface CustomOperator {
|
||||||
|
name: string;
|
||||||
|
sql: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface ComputedColumn {
|
||||||
|
name: string;
|
||||||
|
expression: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
export type VectorMetric = 'l2' | 'cosine' | 'ip';
|
||||||
|
export type ResponseFormat = 'simple' | 'detail' | 'syncfusion';
|
||||||
|
|
||||||
|
/** pgvector KNN search: order by distance between `column` and `vector`. */
|
||||||
|
export interface VectorSearchOption {
|
||||||
|
column: string;
|
||||||
|
vector: number[];
|
||||||
|
metric?: VectorMetric;
|
||||||
|
/** Distance column alias. Default `_distance` */
|
||||||
|
as?: string;
|
||||||
|
direction?: 'asc' | 'desc';
|
||||||
|
}
|
||||||
|
|
||||||
|
/** LEFT JOIN expansion of a relation (X-Expand). */
|
||||||
|
export interface ExpandOption {
|
||||||
|
relation: string;
|
||||||
|
columns?: string[];
|
||||||
|
}
|
||||||
|
|
||||||
|
/** X-Files configuration (Go restheadspec XFiles). Sent as a single JSON header. */
|
||||||
|
export interface XFiles {
|
||||||
|
tablename?: string;
|
||||||
|
schema?: string;
|
||||||
|
primarykey?: string;
|
||||||
|
foreignkey?: string;
|
||||||
|
relatedkey?: string;
|
||||||
|
sort?: string[];
|
||||||
|
prefix?: string;
|
||||||
|
editable?: boolean;
|
||||||
|
recursive?: boolean;
|
||||||
|
expand?: boolean;
|
||||||
|
rownumber?: boolean;
|
||||||
|
skipcount?: boolean;
|
||||||
|
offset?: number;
|
||||||
|
limit?: number;
|
||||||
|
columns?: string[];
|
||||||
|
omit_columns?: string[];
|
||||||
|
cql_columns?: string[];
|
||||||
|
sql_joins?: string[];
|
||||||
|
sql_or?: string[];
|
||||||
|
sql_and?: string[];
|
||||||
|
parenttables?: XFiles[];
|
||||||
|
childtables?: XFiles[];
|
||||||
|
filter_fields?: { field: string; value: string; operator: string }[];
|
||||||
|
cursor_forward?: string;
|
||||||
|
cursor_backward?: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface Options {
|
||||||
|
preload?: PreloadOption[];
|
||||||
|
columns?: string[];
|
||||||
|
omit_columns?: string[];
|
||||||
|
filters?: FilterOption[];
|
||||||
|
sort?: SortOption[];
|
||||||
|
limit?: number;
|
||||||
|
offset?: number;
|
||||||
|
customOperators?: CustomOperator[];
|
||||||
|
computedColumns?: ComputedColumn[];
|
||||||
|
parameters?: Parameter[];
|
||||||
|
cursor_forward?: string;
|
||||||
|
cursor_backward?: string;
|
||||||
|
fetch_row_number?: string;
|
||||||
|
vector_search?: VectorSearchOption;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Options only available to the header-based (restheadspec) protocol. */
|
||||||
|
export interface HeaderSpecOptions extends Options {
|
||||||
|
/** X-Expand: LEFT JOIN relations */
|
||||||
|
expand?: ExpandOption[];
|
||||||
|
/** X-Custom-SQL-Join: raw JOIN clauses */
|
||||||
|
custom_sql_joins?: string[];
|
||||||
|
/** X-Custom-SQL-Or: raw SQL, OR-combined */
|
||||||
|
custom_sql_or?: string[];
|
||||||
|
/** X-SearchCols: columns for multi-column search */
|
||||||
|
search_columns?: string[];
|
||||||
|
/** X-AdvSQL-{col}: column -> SQL expression */
|
||||||
|
advanced_sql?: Record<string, string>;
|
||||||
|
/** X-Clean-JSON */
|
||||||
|
clean_json?: boolean;
|
||||||
|
/** X-Distinct */
|
||||||
|
distinct?: boolean;
|
||||||
|
/** X-SkipCount: skip total count query */
|
||||||
|
skip_count?: boolean;
|
||||||
|
/** X-SkipCache */
|
||||||
|
skip_cache?: boolean;
|
||||||
|
/** X-PKRow: primary key value of a row to fetch */
|
||||||
|
pk_row?: string;
|
||||||
|
/** X-SimpleApi / X-DetailApi / X-Syncfusion */
|
||||||
|
response_format?: ResponseFormat;
|
||||||
|
/** X-Single-Record-As-Object (server default true) */
|
||||||
|
single_record_as_object?: boolean;
|
||||||
|
/** X-Transaction-Atomic */
|
||||||
|
atomic_transaction?: boolean;
|
||||||
|
/** X-Files: single JSON configuration */
|
||||||
|
xfiles?: XFiles;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface RequestBody {
|
||||||
|
operation: Operation;
|
||||||
|
id?: number | string | string[];
|
||||||
|
data?: any | any[];
|
||||||
|
options?: Options;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface Metadata {
|
||||||
|
total: number;
|
||||||
|
count: number;
|
||||||
|
filtered: number;
|
||||||
|
limit: number;
|
||||||
|
offset: number;
|
||||||
|
row_number?: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface APIError {
|
||||||
|
code: string;
|
||||||
|
message: string;
|
||||||
|
details?: any;
|
||||||
|
detail?: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface APIResponse<T = any> {
|
||||||
|
success: boolean;
|
||||||
|
data: T;
|
||||||
|
metadata?: Metadata;
|
||||||
|
error?: APIError;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface Column {
|
||||||
|
name: string;
|
||||||
|
type: string;
|
||||||
|
is_nullable: boolean;
|
||||||
|
is_primary: boolean;
|
||||||
|
is_unique: boolean;
|
||||||
|
has_index: boolean;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface TableMetadata {
|
||||||
|
schema: string;
|
||||||
|
table: string;
|
||||||
|
columns: Column[];
|
||||||
|
relations: string[];
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface ClientConfig {
|
||||||
|
baseUrl: string;
|
||||||
|
token?: string;
|
||||||
|
/** Custom HTTP headers. Token and HeaderSpec query options take precedence. */
|
||||||
|
headers?: Record<string, string>;
|
||||||
|
}
|
||||||
+131
-31
@@ -1,9 +1,11 @@
|
|||||||
|
import { clientCacheKey, clientHeaders, mergeHeaders } from '../common/http';
|
||||||
|
import { b64DecodeUnicode, b64EncodeUnicode } from '@warkypublic/artemis-kit/base64';
|
||||||
import type {
|
import type {
|
||||||
APIResponse,
|
APIResponse,
|
||||||
ClientConfig,
|
ClientConfig,
|
||||||
CustomOperator,
|
CustomOperator,
|
||||||
FilterOption,
|
FilterOption,
|
||||||
Options,
|
HeaderSpecOptions,
|
||||||
PreloadOption,
|
PreloadOption,
|
||||||
SortOption,
|
SortOption,
|
||||||
} from "../common/types";
|
} from "../common/types";
|
||||||
@@ -12,10 +14,7 @@ import type {
|
|||||||
* Encode a value with base64 and ZIP_ prefix for complex header values.
|
* Encode a value with base64 and ZIP_ prefix for complex header values.
|
||||||
*/
|
*/
|
||||||
export function encodeHeaderValue(value: string): string {
|
export function encodeHeaderValue(value: string): string {
|
||||||
if (typeof btoa === "function") {
|
return "ZIP_" + b64EncodeUnicode(value);
|
||||||
return "ZIP_" + btoa(value);
|
|
||||||
}
|
|
||||||
return "ZIP_" + Buffer.from(value, "utf-8").toString("base64");
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -41,10 +40,7 @@ export function decodeHeaderValue(value: string): string {
|
|||||||
}
|
}
|
||||||
|
|
||||||
function decodeBase64(str: string): string {
|
function decodeBase64(str: string): string {
|
||||||
if (typeof atob === "function") {
|
return b64DecodeUnicode(str);
|
||||||
return atob(str);
|
|
||||||
}
|
|
||||||
return Buffer.from(str, "base64").toString("utf-8");
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -63,8 +59,15 @@ function decodeBase64(str: string): string {
|
|||||||
* - X-Fetch-RowNumber: row number fetch
|
* - X-Fetch-RowNumber: row number fetch
|
||||||
* - X-CQL-SEL-{col}: computed columns
|
* - X-CQL-SEL-{col}: computed columns
|
||||||
* - X-Custom-SQL-W: custom operators (AND)
|
* - X-Custom-SQL-W: custom operators (AND)
|
||||||
|
* - X-Preload-Where: where for X-Preload (extra where groups use X-Preload-{n}[-Where])
|
||||||
|
* - X-SpatialFilter-{col} / X-VectorFilter-{col}: JSON {op,value,logic}
|
||||||
|
* - X-Vector-Search-{col|vector|as|dir}: pgvector KNN
|
||||||
|
* - X-Expand, X-Custom-SQL-Join, X-Custom-SQL-Or, X-SearchCols, X-AdvSQL-{col}
|
||||||
|
* - X-Clean-JSON, X-Distinct, X-SkipCount, X-SkipCache, X-PKRow
|
||||||
|
* - X-SimpleApi / X-DetailApi / X-Syncfusion, X-Single-Record-As-Object
|
||||||
|
* - X-Transaction-Atomic, X-Files
|
||||||
*/
|
*/
|
||||||
export function buildHeaders(options: Options): Record<string, string> {
|
export function buildHeaders(options: HeaderSpecOptions): Record<string, string> {
|
||||||
const headers: Record<string, string> = {};
|
const headers: Record<string, string> = {};
|
||||||
|
|
||||||
// Column selection
|
// Column selection
|
||||||
@@ -83,6 +86,17 @@ export function buildHeaders(options: Options): Record<string, string> {
|
|||||||
const op = mapOperatorToHeaderOp(filter.operator);
|
const op = mapOperatorToHeaderOp(filter.operator);
|
||||||
const valueStr = formatFilterValue(filter);
|
const valueStr = formatFilterValue(filter);
|
||||||
|
|
||||||
|
const geoPrefix = geoFilterHeader(filter.operator);
|
||||||
|
if (geoPrefix) {
|
||||||
|
const payload: Record<string, unknown> = {
|
||||||
|
op: filter.operator,
|
||||||
|
value: filter.value,
|
||||||
|
};
|
||||||
|
if (logicOp === "OR") payload.logic = "or";
|
||||||
|
headers[`${geoPrefix}${filter.column}`] = JSON.stringify(payload);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
if (filter.operator === "eq" && logicOp === "AND") {
|
if (filter.operator === "eq" && logicOp === "AND") {
|
||||||
// Simple field filter shorthand
|
// Simple field filter shorthand
|
||||||
headers[`X-FieldFilter-${filter.column}`] = valueStr;
|
headers[`X-FieldFilter-${filter.column}`] = valueStr;
|
||||||
@@ -121,13 +135,94 @@ export function buildHeaders(options: Options): Record<string, string> {
|
|||||||
|
|
||||||
// Preload
|
// Preload
|
||||||
if (options.preload?.length) {
|
if (options.preload?.length) {
|
||||||
const parts = options.preload.map((p: PreloadOption) => {
|
// Go applies X-Preload-Where to every preload in the matching X-Preload header,
|
||||||
if (p.columns?.length) {
|
// so preloads are grouped by where clause.
|
||||||
return `${p.relation}:${p.columns.join(",")}`;
|
const groups = new Map<string, string[]>();
|
||||||
|
for (const p of options.preload) {
|
||||||
|
const spec = p.columns?.length
|
||||||
|
? `${p.relation}:${p.columns.join(",")}`
|
||||||
|
: p.relation;
|
||||||
|
const where = p.where ?? "";
|
||||||
|
groups.set(where, [...(groups.get(where) ?? []), spec]);
|
||||||
|
}
|
||||||
|
let n = 0;
|
||||||
|
for (const [where, specs] of groups) {
|
||||||
|
if (!where) {
|
||||||
|
headers["X-Preload"] = specs.join("|");
|
||||||
|
} else if (!groups.has("") && n === 0) {
|
||||||
|
// X-Preload-Where would also apply to a where-less X-Preload, so only use it alone
|
||||||
|
headers["X-Preload"] = specs.join("|");
|
||||||
|
headers["X-Preload-Where"] = where;
|
||||||
|
n++;
|
||||||
|
} else {
|
||||||
|
n++;
|
||||||
|
headers[`X-Preload-${n}`] = specs.join("|");
|
||||||
|
headers[`X-Preload-${n}-Where`] = where;
|
||||||
}
|
}
|
||||||
return p.relation;
|
}
|
||||||
});
|
}
|
||||||
headers["X-Preload"] = parts.join("|");
|
|
||||||
|
// Expand (LEFT JOIN)
|
||||||
|
if (options.expand?.length) {
|
||||||
|
headers["X-Expand"] = options.expand
|
||||||
|
.map((e) =>
|
||||||
|
e.columns?.length ? `${e.relation}:${e.columns.join(",")}` : e.relation,
|
||||||
|
)
|
||||||
|
.join("|");
|
||||||
|
}
|
||||||
|
|
||||||
|
if (options.custom_sql_joins?.length) {
|
||||||
|
headers["X-Custom-SQL-Join"] = options.custom_sql_joins.join("|");
|
||||||
|
}
|
||||||
|
if (options.custom_sql_or?.length) {
|
||||||
|
headers["X-Custom-SQL-Or"] = options.custom_sql_or.join(" OR ");
|
||||||
|
}
|
||||||
|
if (options.search_columns?.length) {
|
||||||
|
headers["X-SearchCols"] = options.search_columns.join(",");
|
||||||
|
}
|
||||||
|
if (options.advanced_sql) {
|
||||||
|
for (const [col, sql] of Object.entries(options.advanced_sql)) {
|
||||||
|
headers[`X-AdvSQL-${col}`] = sql;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// pgvector KNN search
|
||||||
|
if (options.vector_search) {
|
||||||
|
const vs = options.vector_search;
|
||||||
|
headers[`X-Vector-Search-${vs.column}`] = vs.metric ?? "l2";
|
||||||
|
headers["X-Vector-Search-Vector"] = JSON.stringify(vs.vector);
|
||||||
|
if (vs.as) headers["X-Vector-Search-As"] = vs.as;
|
||||||
|
if (vs.direction) headers["X-Vector-Search-Dir"] = vs.direction;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Flags
|
||||||
|
const flags: [string, boolean | undefined][] = [
|
||||||
|
["X-Clean-JSON", options.clean_json],
|
||||||
|
["X-Distinct", options.distinct],
|
||||||
|
["X-SkipCount", options.skip_count],
|
||||||
|
["X-SkipCache", options.skip_cache],
|
||||||
|
["X-Transaction-Atomic", options.atomic_transaction],
|
||||||
|
["X-Single-Record-As-Object", options.single_record_as_object],
|
||||||
|
];
|
||||||
|
for (const [name, val] of flags) {
|
||||||
|
if (val !== undefined) headers[name] = String(val);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (options.pk_row) {
|
||||||
|
headers["X-PKRow"] = options.pk_row;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (options.response_format) {
|
||||||
|
const formatHeaders = {
|
||||||
|
simple: "X-SimpleApi",
|
||||||
|
detail: "X-DetailApi",
|
||||||
|
syncfusion: "X-Syncfusion",
|
||||||
|
} as const;
|
||||||
|
headers[formatHeaders[options.response_format]] = "true";
|
||||||
|
}
|
||||||
|
|
||||||
|
if (options.xfiles) {
|
||||||
|
headers["X-Files"] = encodeHeaderValue(JSON.stringify(options.xfiles));
|
||||||
}
|
}
|
||||||
|
|
||||||
// Fetch row number
|
// Fetch row number
|
||||||
@@ -153,6 +248,17 @@ export function buildHeaders(options: Options): Record<string, string> {
|
|||||||
return headers;
|
return headers;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const VECTOR_OPS = new Set(["l2_within", "cosine_within", "ip_within"]);
|
||||||
|
|
||||||
|
function geoFilterHeader(operator: string): string | null {
|
||||||
|
const op = operator.toLowerCase();
|
||||||
|
if (VECTOR_OPS.has(op) || op.endsWith("_within")) return "X-VectorFilter-";
|
||||||
|
if (op.startsWith("st_") || op === "bbox" || op === "&&") {
|
||||||
|
return "X-SpatialFilter-";
|
||||||
|
}
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
function mapOperatorToHeaderOp(operator: string): string {
|
function mapOperatorToHeaderOp(operator: string): string {
|
||||||
switch (operator) {
|
switch (operator) {
|
||||||
case "eq":
|
case "eq":
|
||||||
@@ -203,7 +309,7 @@ function formatFilterValue(filter: FilterOption): string {
|
|||||||
const instances = new Map<string, HeaderSpecClient>();
|
const instances = new Map<string, HeaderSpecClient>();
|
||||||
|
|
||||||
export function getHeaderSpecClient(config: ClientConfig): HeaderSpecClient {
|
export function getHeaderSpecClient(config: ClientConfig): HeaderSpecClient {
|
||||||
const key = config.baseUrl;
|
const key = clientCacheKey(config);
|
||||||
let instance = instances.get(key);
|
let instance = instances.get(key);
|
||||||
if (!instance) {
|
if (!instance) {
|
||||||
instance = new HeaderSpecClient(config);
|
instance = new HeaderSpecClient(config);
|
||||||
@@ -222,7 +328,7 @@ export class HeaderSpecClient {
|
|||||||
private config: ClientConfig;
|
private config: ClientConfig;
|
||||||
|
|
||||||
constructor(config: ClientConfig) {
|
constructor(config: ClientConfig) {
|
||||||
this.config = config;
|
this.config = { ...config, headers: { ...config.headers } };
|
||||||
}
|
}
|
||||||
|
|
||||||
private buildUrl(schema: string, entity: string, id?: string): string {
|
private buildUrl(schema: string, entity: string, id?: string): string {
|
||||||
@@ -234,13 +340,7 @@ export class HeaderSpecClient {
|
|||||||
}
|
}
|
||||||
|
|
||||||
private baseHeaders(): Record<string, string> {
|
private baseHeaders(): Record<string, string> {
|
||||||
const headers: Record<string, string> = {
|
return clientHeaders(this.config);
|
||||||
"Content-Type": "application/json",
|
|
||||||
};
|
|
||||||
if (this.config.token) {
|
|
||||||
headers["Authorization"] = `Bearer ${this.config.token}`;
|
|
||||||
}
|
|
||||||
return headers;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
private async fetchWithError<T>(
|
private async fetchWithError<T>(
|
||||||
@@ -290,13 +390,13 @@ export class HeaderSpecClient {
|
|||||||
schema: string,
|
schema: string,
|
||||||
entity: string,
|
entity: string,
|
||||||
id?: string,
|
id?: string,
|
||||||
options?: Options,
|
options?: HeaderSpecOptions,
|
||||||
): Promise<APIResponse<T>> {
|
): Promise<APIResponse<T>> {
|
||||||
const url = this.buildUrl(schema, entity, id);
|
const url = this.buildUrl(schema, entity, id);
|
||||||
const optHeaders = options ? buildHeaders(options) : {};
|
const optHeaders = options ? buildHeaders(options) : {};
|
||||||
return this.fetchWithError<T>(url, {
|
return this.fetchWithError<T>(url, {
|
||||||
method: "GET",
|
method: "GET",
|
||||||
headers: { ...this.baseHeaders(), ...optHeaders },
|
headers: mergeHeaders(this.baseHeaders(), optHeaders),
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -304,13 +404,13 @@ export class HeaderSpecClient {
|
|||||||
schema: string,
|
schema: string,
|
||||||
entity: string,
|
entity: string,
|
||||||
data: any,
|
data: any,
|
||||||
options?: Options,
|
options?: HeaderSpecOptions,
|
||||||
): Promise<APIResponse<T>> {
|
): Promise<APIResponse<T>> {
|
||||||
const url = this.buildUrl(schema, entity);
|
const url = this.buildUrl(schema, entity);
|
||||||
const optHeaders = options ? buildHeaders(options) : {};
|
const optHeaders = options ? buildHeaders(options) : {};
|
||||||
return this.fetchWithError<T>(url, {
|
return this.fetchWithError<T>(url, {
|
||||||
method: "POST",
|
method: "POST",
|
||||||
headers: { ...this.baseHeaders(), ...optHeaders },
|
headers: mergeHeaders(this.baseHeaders(), optHeaders),
|
||||||
body: JSON.stringify(data),
|
body: JSON.stringify(data),
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
@@ -320,13 +420,13 @@ export class HeaderSpecClient {
|
|||||||
entity: string,
|
entity: string,
|
||||||
id: string,
|
id: string,
|
||||||
data: any,
|
data: any,
|
||||||
options?: Options,
|
options?: HeaderSpecOptions,
|
||||||
): Promise<APIResponse<T>> {
|
): Promise<APIResponse<T>> {
|
||||||
const url = this.buildUrl(schema, entity, id);
|
const url = this.buildUrl(schema, entity, id);
|
||||||
const optHeaders = options ? buildHeaders(options) : {};
|
const optHeaders = options ? buildHeaders(options) : {};
|
||||||
return this.fetchWithError<T>(url, {
|
return this.fetchWithError<T>(url, {
|
||||||
method: "PUT",
|
method: "PUT",
|
||||||
headers: { ...this.baseHeaders(), ...optHeaders },
|
headers: mergeHeaders(this.baseHeaders(), optHeaders),
|
||||||
body: JSON.stringify(data),
|
body: JSON.stringify(data),
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
+4
-11
@@ -1,9 +1,10 @@
|
|||||||
|
import { clientCacheKey, clientHeaders } from '../common/http';
|
||||||
import type { ClientConfig, APIResponse, TableMetadata, Options, RequestBody } from '../common/types';
|
import type { ClientConfig, APIResponse, TableMetadata, Options, RequestBody } from '../common/types';
|
||||||
|
|
||||||
const instances = new Map<string, ResolveSpecClient>();
|
const instances = new Map<string, ResolveSpecClient>();
|
||||||
|
|
||||||
export function getResolveSpecClient(config: ClientConfig): ResolveSpecClient {
|
export function getResolveSpecClient(config: ClientConfig): ResolveSpecClient {
|
||||||
const key = config.baseUrl;
|
const key = clientCacheKey(config);
|
||||||
let instance = instances.get(key);
|
let instance = instances.get(key);
|
||||||
if (!instance) {
|
if (!instance) {
|
||||||
instance = new ResolveSpecClient(config);
|
instance = new ResolveSpecClient(config);
|
||||||
@@ -16,7 +17,7 @@ export class ResolveSpecClient {
|
|||||||
private config: ClientConfig;
|
private config: ClientConfig;
|
||||||
|
|
||||||
constructor(config: ClientConfig) {
|
constructor(config: ClientConfig) {
|
||||||
this.config = config;
|
this.config = { ...config, headers: { ...config.headers } };
|
||||||
}
|
}
|
||||||
|
|
||||||
private buildUrl(schema: string, entity: string, id?: string): string {
|
private buildUrl(schema: string, entity: string, id?: string): string {
|
||||||
@@ -28,15 +29,7 @@ export class ResolveSpecClient {
|
|||||||
}
|
}
|
||||||
|
|
||||||
private baseHeaders(): HeadersInit {
|
private baseHeaders(): HeadersInit {
|
||||||
const headers: Record<string, string> = {
|
return clientHeaders(this.config);
|
||||||
'Content-Type': 'application/json',
|
|
||||||
};
|
|
||||||
|
|
||||||
if (this.config.token) {
|
|
||||||
headers['Authorization'] = `Bearer ${this.config.token}`;
|
|
||||||
}
|
|
||||||
|
|
||||||
return headers;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
private async fetchWithError<T>(url: string, options: RequestInit): Promise<APIResponse<T>> {
|
private async fetchWithError<T>(url: string, options: RequestInit): Promise<APIResponse<T>> {
|
||||||
@@ -14,7 +14,7 @@ export default defineConfig({
|
|||||||
fileName: (format) => `index.${format === 'es' ? 'js' : 'cjs'}`,
|
fileName: (format) => `index.${format === 'es' ? 'js' : 'cjs'}`,
|
||||||
},
|
},
|
||||||
rollupOptions: {
|
rollupOptions: {
|
||||||
external: ['uuid', 'semver'],
|
external: ['uuid', 'semver', '@warkypublic/artemis-kit/base64'],
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
@@ -0,0 +1,6 @@
|
|||||||
|
__pycache__/
|
||||||
|
*.egg-info/
|
||||||
|
.venv/
|
||||||
|
dist/
|
||||||
|
.pytest_cache/
|
||||||
|
.coverage
|
||||||
@@ -0,0 +1,142 @@
|
|||||||
|
# resolvespec (Python)
|
||||||
|
|
||||||
|
Python client for ResolveSpec REST, HeaderSpec (restheadspec), FunctionSpec and WebSocketSpec. Port of `resolvespec-js`.
|
||||||
|
|
||||||
|
- Python >= 3.11, `httpx` (REST, sync + async), `websockets` (WS, async)
|
||||||
|
- Options/filters/sorts are plain dicts using the wire key names (`TypedDict` hints in `resolvespec.types`)
|
||||||
|
|
||||||
|
```
|
||||||
|
pip install resolvespec
|
||||||
|
```
|
||||||
|
|
||||||
|
## Clients
|
||||||
|
|
||||||
|
| Protocol | Sync | Async | Transport |
|
||||||
|
|---|---|---|---|
|
||||||
|
| ResolveSpec | `ResolveSpecClient` | `AsyncResolveSpecClient` | POST + JSON body `{operation, id, data, options}` |
|
||||||
|
| HeaderSpec | `HeaderSpecClient` | `AsyncHeaderSpecClient` | GET/POST/PUT/DELETE, options as `X-*` headers |
|
||||||
|
| FunctionSpec | `FuncSpecClient` | `AsyncFuncSpecClient` | user-defined SQL endpoints; params via query string + `X-*` headers |
|
||||||
|
| WebSocketSpec | - | `WebSocketClient` | WebSocket JSON messages |
|
||||||
|
|
||||||
|
Constructor (REST): `Client(base_url, token=None, headers=None, timeout=30.0)`
|
||||||
|
|
||||||
|
- `token` -> `Authorization: Bearer`; wins over `headers`
|
||||||
|
- `headers`: custom headers, merged case-insensitively; snapshot at construction
|
||||||
|
- Sync: context manager / `close()`. Async: `async with` / `await aclose()`
|
||||||
|
- Cached sync factories: `get_resolvespec_client()`, `get_headerspec_client()` (same args -> same instance)
|
||||||
|
|
||||||
|
## ResolveSpec
|
||||||
|
|
||||||
|
URL: `{base}/{schema}/{entity}[/{id}]`
|
||||||
|
|
||||||
|
| Method | Signature |
|
||||||
|
|---|---|
|
||||||
|
| `get_metadata` | `(schema, entity)` (GET) |
|
||||||
|
| `read` | `(schema, entity, id=None, options=None)` |
|
||||||
|
| `create` | `(schema, entity, data, options=None)` |
|
||||||
|
| `update` | `(schema, entity, data, id=None, options=None)` |
|
||||||
|
| `delete` | `(schema, entity, id)` |
|
||||||
|
|
||||||
|
`id`: int/str -> URL path; `list[str]` -> body `id`.
|
||||||
|
Returns `{"success", "data", "metadata"?, "error"?}`.
|
||||||
|
|
||||||
|
## HeaderSpec
|
||||||
|
|
||||||
|
| Method | HTTP | Signature |
|
||||||
|
|---|---|---|
|
||||||
|
| `read` | GET | `(schema, entity, id=None, options=None)` |
|
||||||
|
| `create` | POST | `(schema, entity, data, options=None)` |
|
||||||
|
| `update` | PUT | `(schema, entity, id, data, options=None)` |
|
||||||
|
| `delete` | DELETE | `(schema, entity, id)` |
|
||||||
|
|
||||||
|
Response metadata derived from `Content-Range` (`offset-end/total`) and `X-Limit`.
|
||||||
|
`build_headers(options)`, `encode_header_value()` / `decode_header_value()` (`ZIP_` / `__` base64) are exported.
|
||||||
|
|
||||||
|
### Option -> header
|
||||||
|
|
||||||
|
| Option | Header |
|
||||||
|
|---|---|
|
||||||
|
| `columns` / `omit_columns` | `X-Select-Fields` / `X-Not-Select-Fields` |
|
||||||
|
| filter `eq` + AND | `X-FieldFilter-{col}` |
|
||||||
|
| filter AND / OR | `X-SearchOp-{op}-{col}` / `X-SearchOr-{op}-{col}` |
|
||||||
|
| spatial (`st_*`, `bbox`) / vector (`*_within`) filter | `X-SpatialFilter-{col}` / `X-VectorFilter-{col}` (JSON) |
|
||||||
|
| `sort` | `X-Sort` (`+col,-col`) |
|
||||||
|
| `limit` / `offset` | `X-Limit` / `X-Offset` |
|
||||||
|
| `cursor_forward` / `cursor_backward` | `X-Cursor-Forward` / `X-Cursor-Backward` |
|
||||||
|
| `preload` | `X-Preload` (`Rel:c1,c2\|Rel2`), `X-Preload-Where`, `X-Preload-{n}[-Where]` |
|
||||||
|
| `expand` | `X-Expand` |
|
||||||
|
| `custom_sql_joins` / `custom_sql_or` | `X-Custom-SQL-Join` / `X-Custom-SQL-Or` |
|
||||||
|
| `search_columns` | `X-SearchCols` |
|
||||||
|
| `advanced_sql` | `X-AdvSQL-{col}` |
|
||||||
|
| `computedColumns` | `X-CQL-SEL-{name}` |
|
||||||
|
| `customOperators` | `X-Custom-SQL-W` (AND-joined) |
|
||||||
|
| `vector_search` | `X-Vector-Search-{col}`, `-Vector`, `-As`, `-Dir` |
|
||||||
|
| `fetch_row_number` | `X-Fetch-RowNumber` |
|
||||||
|
| `clean_json`, `distinct`, `skip_count`, `skip_cache`, `atomic_transaction`, `single_record_as_object` | `X-Clean-JSON`, `X-Distinct`, `X-SkipCount`, `X-SkipCache`, `X-Transaction-Atomic`, `X-Single-Record-As-Object` |
|
||||||
|
| `pk_row` | `X-PKRow` |
|
||||||
|
| `response_format` (`simple`/`detail`/`syncfusion`) | `X-SimpleApi` / `X-DetailApi` / `X-Syncfusion` |
|
||||||
|
| `xfiles` | `X-Files` (`ZIP_` base64 JSON) |
|
||||||
|
|
||||||
|
Filter operator -> header op: `eq equals`, `neq notequals`, `gt greaterthan`, `gte greaterthanorequal`, `lt lessthan`, `lte lessthanorequal`, `like/ilike/contains contains`, `startswith beginswith`, `endswith`, `in`, `between`, `between_inclusive betweeninclusive`, `is_null empty`, `is_not_null notempty`.
|
||||||
|
|
||||||
|
## FunctionSpec
|
||||||
|
|
||||||
|
Routes are defined by the server app, so calls take a `path`. The server never reads a request body.
|
||||||
|
|
||||||
|
| Method | Server handler | Result |
|
||||||
|
|---|---|---|
|
||||||
|
| `query(path, params=None, options=None, *, method="GET")` | `SqlQuery` (single record) | `{success, data}` |
|
||||||
|
| `query_list(path, params=None, options=None, *, method="GET")` | `SqlQueryList` | `{success, data, metadata}` (from `Content-Range: items a-b/total`) |
|
||||||
|
|
||||||
|
- `params` -> query string. `bool` -> `true/false`, `None` skipped, `list` -> repeated key (server: `IN` filter). `p-` prefixed names are substituted into the SQL.
|
||||||
|
- `options` -> `X-*` headers. Query values override headers of the same name.
|
||||||
|
- 206 Partial Content (more rows than returned) is treated as success.
|
||||||
|
|
||||||
|
| Option | Header |
|
||||||
|
|---|---|
|
||||||
|
| `filters` (`eq`+AND) | `X-FieldFilter-{col}` |
|
||||||
|
| `filters` (other) | `X-SearchOp-{op}-{col}` / `X-SearchOr-{op}-{col}` |
|
||||||
|
| `search_filters` `{col: text}` | `X-SearchFilter-{col}` (ILIKE) |
|
||||||
|
| `custom_sql_where` / `custom_sql_or` | `X-Custom-SQL-W` / `X-Custom-SQL-Or` |
|
||||||
|
| `sort` | `X-Sort` as SQL terms: `col ASC,col DESC` |
|
||||||
|
| `limit` / `offset` | `X-Limit` / `X-Offset` |
|
||||||
|
| `distinct`, `skip_count`, `skip_cache` | `X-Distinct`, `X-SkipCount`, `X-SkipCache` |
|
||||||
|
| `response_format` | `X-SimpleApi` / `X-DetailApi` / `X-Syncfusion` (`data` shape changes: array / `{items,...}` / `{result,count}`) |
|
||||||
|
|
||||||
|
Server limits:
|
||||||
|
- `sort` goes verbatim into `ORDER BY`; `-col` (restheadspec style) does **not** mean DESC.
|
||||||
|
- `X-Select-Fields` / `X-Not-Select-Fields` are no-ops server-side, so not exposed.
|
||||||
|
- One search operator per column; same column twice keeps the last.
|
||||||
|
- Values starting with `ZIP_` / `__` are base64-decoded by the server; such plaintext cannot be sent.
|
||||||
|
- Non-ASCII / control-char values are sent `ZIP_`-encoded automatically.
|
||||||
|
|
||||||
|
## WebSocketSpec
|
||||||
|
|
||||||
|
`WebSocketClient(url, *, reconnect=True, reconnect_interval=3.0, max_reconnect_attempts=10, heartbeat_interval=30.0, request_timeout=30.0, subscribe_timeout=10.0, headers=None)`
|
||||||
|
|
||||||
|
| Method | Notes |
|
||||||
|
|---|---|
|
||||||
|
| `connect()` / `close()` | also `async with` |
|
||||||
|
| `request(operation, entity, *, schema, record_id, data, options)` | returns response `data` |
|
||||||
|
| `read(entity, *, schema, record_id, filters, columns, sort, preload, limit, offset)` | |
|
||||||
|
| `create(entity, data, *, schema)` | |
|
||||||
|
| `update(entity, id, data, *, schema)` | |
|
||||||
|
| `delete(entity, id, *, schema)` | |
|
||||||
|
| `meta(entity, *, schema)` | |
|
||||||
|
| `subscribe(entity, callback, *, schema, filters)` | returns subscription id; callback gets notification dict (sync or async) |
|
||||||
|
| `unsubscribe(subscription_id)` | |
|
||||||
|
| `on(event, cb)` / `off(event)` | events: `connect`, `disconnect`, `error`, `message`, `state_change` |
|
||||||
|
| `state`, `is_connected()`, `get_subscriptions()` | |
|
||||||
|
|
||||||
|
Auto-reconnect does not restore subscriptions; re-subscribe on `connect`.
|
||||||
|
|
||||||
|
## Errors
|
||||||
|
|
||||||
|
`ResolveSpecError(message, status_code, code, details)` on non-2xx (REST) or failed response / timeout / not connected (WS).
|
||||||
|
|
||||||
|
## Dev
|
||||||
|
|
||||||
|
```
|
||||||
|
pip install -e '.[dev]'
|
||||||
|
pytest
|
||||||
|
```
|
||||||
@@ -0,0 +1,24 @@
|
|||||||
|
[build-system]
|
||||||
|
requires = ["hatchling"]
|
||||||
|
build-backend = "hatchling.build"
|
||||||
|
|
||||||
|
[project]
|
||||||
|
name = "resolvespec"
|
||||||
|
version = "1.0.0"
|
||||||
|
description = "Python client for ResolveSpec REST, HeaderSpec and WebSocket APIs"
|
||||||
|
readme = "README.md"
|
||||||
|
requires-python = ">=3.11"
|
||||||
|
license = { text = "MIT" }
|
||||||
|
authors = [{ name = "Hein (Warkanum) Puth" }]
|
||||||
|
keywords = ["resolvespec", "headerspec", "websocket", "rest-client", "api-client"]
|
||||||
|
dependencies = ["httpx>=0.27", "websockets>=13"]
|
||||||
|
|
||||||
|
[project.optional-dependencies]
|
||||||
|
dev = ["pytest>=8", "pytest-asyncio>=0.23", "pytest-cov"]
|
||||||
|
|
||||||
|
[tool.hatch.build.targets.wheel]
|
||||||
|
packages = ["src/resolvespec"]
|
||||||
|
|
||||||
|
[tool.pytest.ini_options]
|
||||||
|
testpaths = ["tests"]
|
||||||
|
asyncio_mode = "auto"
|
||||||
@@ -0,0 +1,43 @@
|
|||||||
|
"""ResolveSpec Python client: REST (ResolveSpec), HeaderSpec and WebSocketSpec."""
|
||||||
|
from typing import Mapping, Optional
|
||||||
|
|
||||||
|
from .headerspec import (
|
||||||
|
AsyncHeaderSpecClient,
|
||||||
|
HeaderSpecClient,
|
||||||
|
build_headers,
|
||||||
|
decode_header_value,
|
||||||
|
encode_header_value,
|
||||||
|
)
|
||||||
|
from .funcspec import AsyncFuncSpecClient, FuncSpecClient
|
||||||
|
from .http import ResolveSpecError, merge_headers
|
||||||
|
from .resolvespec import AsyncResolveSpecClient, ResolveSpecClient
|
||||||
|
from .types import * # noqa: F401,F403
|
||||||
|
from .websocket import Subscription, WebSocketClient
|
||||||
|
|
||||||
|
|
||||||
|
def _cache_key(base_url: str, token: Optional[str], headers: Optional[Mapping[str, str]]):
|
||||||
|
return (
|
||||||
|
base_url,
|
||||||
|
token,
|
||||||
|
tuple(sorted((k.lower(), v) for k, v in (headers or {}).items())),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
_resolvespec: dict = {}
|
||||||
|
_headerspec: dict = {}
|
||||||
|
|
||||||
|
|
||||||
|
def get_resolvespec_client(base_url: str, token: Optional[str] = None, headers: Optional[Mapping[str, str]] = None) -> ResolveSpecClient:
|
||||||
|
"""Cached sync client, keyed by base_url + token + headers (case-insensitive names)."""
|
||||||
|
key = _cache_key(base_url, token, headers)
|
||||||
|
if key not in _resolvespec:
|
||||||
|
_resolvespec[key] = ResolveSpecClient(base_url, token, headers)
|
||||||
|
return _resolvespec[key]
|
||||||
|
|
||||||
|
|
||||||
|
def get_headerspec_client(base_url: str, token: Optional[str] = None, headers: Optional[Mapping[str, str]] = None) -> HeaderSpecClient:
|
||||||
|
"""Cached sync client, keyed by base_url + token + headers (case-insensitive names)."""
|
||||||
|
key = _cache_key(base_url, token, headers)
|
||||||
|
if key not in _headerspec:
|
||||||
|
_headerspec[key] = HeaderSpecClient(base_url, token, headers)
|
||||||
|
return _headerspec[key]
|
||||||
@@ -0,0 +1,197 @@
|
|||||||
|
"""FunctionSpec client: calls user-defined SQL endpoints (Go pkg/funcspec).
|
||||||
|
|
||||||
|
Routes are defined by the server application, so calls take a `path`.
|
||||||
|
Parameters are sent as query string values and/or `X-*` headers; the server never
|
||||||
|
reads a request body. Query-string values override headers of the same name.
|
||||||
|
|
||||||
|
Server behaviour worth knowing (pkg/funcspec):
|
||||||
|
- `sort` is inserted raw into ORDER BY, so it must be SQL (`col DESC`), not `-col`.
|
||||||
|
- Field selection (`X-Select-Fields`) is a no-op server-side, so it is not exposed.
|
||||||
|
- Only one search operator per column is kept.
|
||||||
|
- Values starting with `ZIP_` or `__` are base64-decoded by the server (even after our
|
||||||
|
own encoding), so such plaintext values cannot be sent faithfully.
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import re
|
||||||
|
from typing import Any, Dict, List, Mapping, Optional
|
||||||
|
|
||||||
|
import httpx
|
||||||
|
|
||||||
|
from .headerspec import _OPERATOR_MAP, _bool, _filter_value, encode_header_value
|
||||||
|
from .http import client_headers, error_from, merge_headers, parse_json
|
||||||
|
from .types import APIResponse, FuncSpecOptions
|
||||||
|
|
||||||
|
Params = Mapping[str, Any]
|
||||||
|
|
||||||
|
_CONTENT_RANGE = re.compile(r"(\d+)-(\d+)/(\d+)")
|
||||||
|
|
||||||
|
|
||||||
|
def _safe(value: str) -> str:
|
||||||
|
"""Encode values that are unsafe as raw header/query text (non-ASCII, control chars, edge spaces)."""
|
||||||
|
if not value.isascii() or not value.isprintable() or value != value.strip():
|
||||||
|
return encode_header_value(value)
|
||||||
|
return value
|
||||||
|
|
||||||
|
|
||||||
|
def build_headers(options: Mapping[str, Any]) -> Dict[str, str]:
|
||||||
|
"""Build the X-* headers understood by funcspec.ParseParameters."""
|
||||||
|
h: Dict[str, str] = {}
|
||||||
|
o = options
|
||||||
|
|
||||||
|
for f in o.get("filters") or []:
|
||||||
|
operator = f["operator"]
|
||||||
|
logic = f.get("logic_operator") or "AND"
|
||||||
|
value = _safe(_filter_value(f))
|
||||||
|
if operator == "eq" and logic == "AND":
|
||||||
|
h[f"X-FieldFilter-{f['column']}"] = value
|
||||||
|
else:
|
||||||
|
kind = "X-SearchOr" if logic == "OR" else "X-SearchOp"
|
||||||
|
h[f"{kind}-{_OPERATOR_MAP.get(operator, operator)}-{f['column']}"] = value
|
||||||
|
|
||||||
|
for col, text in (o.get("search_filters") or {}).items():
|
||||||
|
h[f"X-SearchFilter-{col}"] = _safe(str(text)) # CAST(col AS TEXT) ILIKE %text%
|
||||||
|
|
||||||
|
if o.get("custom_sql_where"):
|
||||||
|
h["X-Custom-SQL-W"] = _safe(o["custom_sql_where"])
|
||||||
|
if o.get("custom_sql_or"):
|
||||||
|
h["X-Custom-SQL-Or"] = _safe(o["custom_sql_or"])
|
||||||
|
|
||||||
|
if o.get("sort"):
|
||||||
|
h["X-Sort"] = _safe(",".join(_sort_term(s) for s in o["sort"]))
|
||||||
|
if o.get("limit") is not None:
|
||||||
|
h["X-Limit"] = str(o["limit"])
|
||||||
|
if o.get("offset") is not None:
|
||||||
|
h["X-Offset"] = str(o["offset"])
|
||||||
|
|
||||||
|
for name, key in (("X-Distinct", "distinct"), ("X-SkipCount", "skip_count"), ("X-SkipCache", "skip_cache")):
|
||||||
|
if o.get(key) is not None:
|
||||||
|
h[name] = _bool(o[key])
|
||||||
|
|
||||||
|
fmt = o.get("response_format")
|
||||||
|
if fmt:
|
||||||
|
h[{"simple": "X-SimpleApi", "detail": "X-DetailApi", "syncfusion": "X-Syncfusion"}[fmt]] = "true"
|
||||||
|
return h
|
||||||
|
|
||||||
|
|
||||||
|
def _sort_term(s: Mapping[str, str]) -> str:
|
||||||
|
# funcspec puts this verbatim into ORDER BY
|
||||||
|
return f"{s['column']} {'DESC' if s.get('direction', 'asc').upper() == 'DESC' else 'ASC'}"
|
||||||
|
|
||||||
|
|
||||||
|
def build_query(params: Optional[Params]) -> Dict[str, Any]:
|
||||||
|
"""Query-string values: bools -> true/false, lists -> repeated keys (server: IN filter)."""
|
||||||
|
out: Dict[str, Any] = {}
|
||||||
|
for k, v in (params or {}).items():
|
||||||
|
if v is None:
|
||||||
|
continue
|
||||||
|
if isinstance(v, (list, tuple)):
|
||||||
|
out[k] = [_safe(_q(x)) for x in v]
|
||||||
|
else:
|
||||||
|
out[k] = _safe(_q(v))
|
||||||
|
return out
|
||||||
|
|
||||||
|
|
||||||
|
def _q(v: Any) -> str:
|
||||||
|
return _bool(v) if isinstance(v, bool) else str(v)
|
||||||
|
|
||||||
|
|
||||||
|
def _metadata(response: httpx.Response, options: Optional[Mapping[str, Any]]) -> Dict[str, int]:
|
||||||
|
"""Content-Range is `items {offset}-{offset+len}/{total}`."""
|
||||||
|
m = _CONTENT_RANGE.search(response.headers.get("content-range", ""))
|
||||||
|
start, end, total = (int(x) for x in m.groups()) if m else (0, 0, 0)
|
||||||
|
return {
|
||||||
|
"total": total,
|
||||||
|
"count": end - start,
|
||||||
|
"filtered": total,
|
||||||
|
"offset": start,
|
||||||
|
"limit": int((options or {}).get("limit") or 0),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _wrap(response: httpx.Response, options: Optional[Mapping[str, Any]], with_metadata: bool) -> APIResponse:
|
||||||
|
data = parse_json(response)
|
||||||
|
if not response.is_success: # 206 Partial Content is success
|
||||||
|
raise error_from(response, data)
|
||||||
|
result: APIResponse = {"success": True, "data": data}
|
||||||
|
if with_metadata:
|
||||||
|
result["metadata"] = _metadata(response, options)
|
||||||
|
return result
|
||||||
|
|
||||||
|
|
||||||
|
class _Base:
|
||||||
|
def __init__(
|
||||||
|
self,
|
||||||
|
base_url: str,
|
||||||
|
token: Optional[str] = None,
|
||||||
|
headers: Optional[Mapping[str, str]] = None,
|
||||||
|
timeout: Optional[float] = 30.0,
|
||||||
|
):
|
||||||
|
self.base_url = base_url
|
||||||
|
self.token = token
|
||||||
|
self.headers = dict(headers or {}) # snapshot
|
||||||
|
self.timeout = timeout
|
||||||
|
|
||||||
|
def _req(self, method: str, path: str, params: Optional[Params], options: Optional[Mapping[str, Any]]):
|
||||||
|
url = f"{self.base_url.rstrip('/')}/{path.lstrip('/')}"
|
||||||
|
headers = merge_headers(
|
||||||
|
client_headers(self.token, self.headers),
|
||||||
|
build_headers(options) if options else {},
|
||||||
|
)
|
||||||
|
return method.upper(), url, headers, build_query(params)
|
||||||
|
|
||||||
|
|
||||||
|
class FuncSpecClient(_Base):
|
||||||
|
"""Synchronous client. Use as a context manager or call close()."""
|
||||||
|
|
||||||
|
def __init__(self, *args: Any, transport: Optional[httpx.BaseTransport] = None, **kwargs: Any):
|
||||||
|
super().__init__(*args, **kwargs)
|
||||||
|
self._http = httpx.Client(timeout=self.timeout, transport=transport)
|
||||||
|
|
||||||
|
def close(self) -> None:
|
||||||
|
self._http.close()
|
||||||
|
|
||||||
|
def __enter__(self) -> "FuncSpecClient":
|
||||||
|
return self
|
||||||
|
|
||||||
|
def __exit__(self, *exc: Any) -> None:
|
||||||
|
self.close()
|
||||||
|
|
||||||
|
def _send(self, req, options, with_metadata) -> APIResponse:
|
||||||
|
method, url, headers, query = req
|
||||||
|
return _wrap(self._http.request(method, url, headers=headers, params=query), options, with_metadata)
|
||||||
|
|
||||||
|
def query(self, path: str, params: Optional[Params] = None, options: Optional[FuncSpecOptions] = None, *, method: str = "GET") -> APIResponse:
|
||||||
|
"""Single-record endpoint (Handler.SqlQuery). `data` is the row object."""
|
||||||
|
return self._send(self._req(method, path, params, options), options, False)
|
||||||
|
|
||||||
|
def query_list(self, path: str, params: Optional[Params] = None, options: Optional[FuncSpecOptions] = None, *, method: str = "GET") -> APIResponse:
|
||||||
|
"""List endpoint (Handler.SqlQueryList). Adds `metadata` from Content-Range."""
|
||||||
|
return self._send(self._req(method, path, params, options), options, True)
|
||||||
|
|
||||||
|
|
||||||
|
class AsyncFuncSpecClient(_Base):
|
||||||
|
"""Asyncio client. Use as an async context manager or await aclose()."""
|
||||||
|
|
||||||
|
def __init__(self, *args: Any, transport: Optional[httpx.AsyncBaseTransport] = None, **kwargs: Any):
|
||||||
|
super().__init__(*args, **kwargs)
|
||||||
|
self._http = httpx.AsyncClient(timeout=self.timeout, transport=transport)
|
||||||
|
|
||||||
|
async def aclose(self) -> None:
|
||||||
|
await self._http.aclose()
|
||||||
|
|
||||||
|
async def __aenter__(self) -> "AsyncFuncSpecClient":
|
||||||
|
return self
|
||||||
|
|
||||||
|
async def __aexit__(self, *exc: Any) -> None:
|
||||||
|
await self.aclose()
|
||||||
|
|
||||||
|
async def _send(self, req, options, with_metadata) -> APIResponse:
|
||||||
|
method, url, headers, query = req
|
||||||
|
return _wrap(await self._http.request(method, url, headers=headers, params=query), options, with_metadata)
|
||||||
|
|
||||||
|
async def query(self, path: str, params: Optional[Params] = None, options: Optional[FuncSpecOptions] = None, *, method: str = "GET") -> APIResponse:
|
||||||
|
return await self._send(self._req(method, path, params, options), options, False)
|
||||||
|
|
||||||
|
async def query_list(self, path: str, params: Optional[Params] = None, options: Optional[FuncSpecOptions] = None, *, method: str = "GET") -> APIResponse:
|
||||||
|
return await self._send(self._req(method, path, params, options), options, True)
|
||||||
@@ -0,0 +1,336 @@
|
|||||||
|
"""HeaderSpec client: query options sent as HTTP headers (Go restheadspec).
|
||||||
|
|
||||||
|
Methods: GET=read, POST=create, PUT=update, DELETE=delete.
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import base64
|
||||||
|
import json
|
||||||
|
import re
|
||||||
|
from typing import Any, Dict, Mapping, Optional
|
||||||
|
|
||||||
|
import httpx
|
||||||
|
|
||||||
|
from .http import build_url, client_headers, error_from, merge_headers, parse_json
|
||||||
|
from .types import APIResponse, FilterOption, HeaderSpecOptions
|
||||||
|
|
||||||
|
_PREFIXES = ("ZIP_", "__")
|
||||||
|
|
||||||
|
_OPERATOR_MAP = {
|
||||||
|
"eq": "equals",
|
||||||
|
"neq": "notequals",
|
||||||
|
"gt": "greaterthan",
|
||||||
|
"gte": "greaterthanorequal",
|
||||||
|
"lt": "lessthan",
|
||||||
|
"lte": "lessthanorequal",
|
||||||
|
"like": "contains",
|
||||||
|
"ilike": "contains",
|
||||||
|
"contains": "contains",
|
||||||
|
"startswith": "beginswith",
|
||||||
|
"endswith": "endswith",
|
||||||
|
"in": "in",
|
||||||
|
"between": "between",
|
||||||
|
"between_inclusive": "betweeninclusive",
|
||||||
|
"is_null": "empty",
|
||||||
|
"is_not_null": "notempty",
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def encode_header_value(value: str) -> str:
|
||||||
|
"""Base64 (UTF-8) with ZIP_ prefix, for complex header values."""
|
||||||
|
return "ZIP_" + base64.b64encode(value.encode("utf-8")).decode("ascii")
|
||||||
|
|
||||||
|
|
||||||
|
def decode_header_value(value: str) -> str:
|
||||||
|
"""Decode a value that may carry a ZIP_ or __ base64 prefix (nested allowed)."""
|
||||||
|
code = value
|
||||||
|
for prefix in _PREFIXES:
|
||||||
|
if code.startswith(prefix):
|
||||||
|
b64 = re.sub(r"[\n\r ]", "", code[len(prefix):])
|
||||||
|
b64 += "=" * (-len(b64) % 4)
|
||||||
|
code = base64.b64decode(b64).decode("utf-8")
|
||||||
|
break
|
||||||
|
if code.startswith(_PREFIXES):
|
||||||
|
code = decode_header_value(code)
|
||||||
|
return code
|
||||||
|
|
||||||
|
|
||||||
|
def _geo_header(operator: str) -> Optional[str]:
|
||||||
|
op = operator.lower()
|
||||||
|
if op.endswith("_within"):
|
||||||
|
return "X-VectorFilter-"
|
||||||
|
if op.startswith("st_") or op in ("bbox", "&&"):
|
||||||
|
return "X-SpatialFilter-"
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def _filter_value(f: FilterOption) -> str:
|
||||||
|
v = f.get("value")
|
||||||
|
if v is None:
|
||||||
|
return ""
|
||||||
|
if isinstance(v, (list, tuple)):
|
||||||
|
return ",".join(_scalar(x) for x in v)
|
||||||
|
return _scalar(v)
|
||||||
|
|
||||||
|
|
||||||
|
def _scalar(v: Any) -> str:
|
||||||
|
if isinstance(v, bool): # match JS String(true)
|
||||||
|
return "true" if v else "false"
|
||||||
|
return str(v)
|
||||||
|
|
||||||
|
|
||||||
|
def _bool(v: bool) -> str:
|
||||||
|
return "true" if v else "false"
|
||||||
|
|
||||||
|
|
||||||
|
def _preload_spec(p: Mapping[str, Any]) -> str:
|
||||||
|
cols = p.get("columns")
|
||||||
|
return f"{p['relation']}:{','.join(cols)}" if cols else p["relation"]
|
||||||
|
|
||||||
|
|
||||||
|
def build_headers(options: HeaderSpecOptions) -> Dict[str, str]:
|
||||||
|
"""Build restheadspec HTTP headers from options. See README for the mapping."""
|
||||||
|
h: Dict[str, str] = {}
|
||||||
|
o = options
|
||||||
|
|
||||||
|
if o.get("columns"):
|
||||||
|
h["X-Select-Fields"] = ",".join(o["columns"])
|
||||||
|
if o.get("omit_columns"):
|
||||||
|
h["X-Not-Select-Fields"] = ",".join(o["omit_columns"])
|
||||||
|
|
||||||
|
for f in o.get("filters") or []:
|
||||||
|
logic = f.get("logic_operator") or "AND"
|
||||||
|
operator = f["operator"]
|
||||||
|
op = _OPERATOR_MAP.get(operator, operator)
|
||||||
|
value = _filter_value(f)
|
||||||
|
geo = _geo_header(operator)
|
||||||
|
if geo:
|
||||||
|
payload: Dict[str, Any] = {"op": operator, "value": f.get("value")}
|
||||||
|
if logic == "OR":
|
||||||
|
payload["logic"] = "or"
|
||||||
|
h[f"{geo}{f['column']}"] = json.dumps(payload, separators=(",", ":"))
|
||||||
|
elif operator == "eq" and logic == "AND":
|
||||||
|
h[f"X-FieldFilter-{f['column']}"] = value
|
||||||
|
elif logic == "OR":
|
||||||
|
h[f"X-SearchOr-{op}-{f['column']}"] = value
|
||||||
|
else:
|
||||||
|
h[f"X-SearchOp-{op}-{f['column']}"] = value
|
||||||
|
|
||||||
|
if o.get("sort"):
|
||||||
|
h["X-Sort"] = ",".join(
|
||||||
|
("-" if s["direction"].upper() == "DESC" else "+") + s["column"] for s in o["sort"]
|
||||||
|
)
|
||||||
|
|
||||||
|
if o.get("limit") is not None:
|
||||||
|
h["X-Limit"] = str(o["limit"])
|
||||||
|
if o.get("offset") is not None:
|
||||||
|
h["X-Offset"] = str(o["offset"])
|
||||||
|
if o.get("cursor_forward"):
|
||||||
|
h["X-Cursor-Forward"] = o["cursor_forward"]
|
||||||
|
if o.get("cursor_backward"):
|
||||||
|
h["X-Cursor-Backward"] = o["cursor_backward"]
|
||||||
|
|
||||||
|
if o.get("preload"):
|
||||||
|
# Go applies X-Preload-Where to every preload in the matching X-Preload header,
|
||||||
|
# so preloads are grouped by where clause.
|
||||||
|
groups: Dict[str, list] = {}
|
||||||
|
for p in o["preload"]:
|
||||||
|
groups.setdefault(p.get("where") or "", []).append(_preload_spec(p))
|
||||||
|
n = 0
|
||||||
|
for where, specs in groups.items():
|
||||||
|
if not where:
|
||||||
|
h["X-Preload"] = "|".join(specs)
|
||||||
|
elif "" not in groups and n == 0:
|
||||||
|
# X-Preload-Where would also apply to a where-less X-Preload, so only use it alone
|
||||||
|
h["X-Preload"] = "|".join(specs)
|
||||||
|
h["X-Preload-Where"] = where
|
||||||
|
n += 1
|
||||||
|
else:
|
||||||
|
n += 1
|
||||||
|
h[f"X-Preload-{n}"] = "|".join(specs)
|
||||||
|
h[f"X-Preload-{n}-Where"] = where
|
||||||
|
|
||||||
|
if o.get("expand"):
|
||||||
|
h["X-Expand"] = "|".join(_preload_spec(e) for e in o["expand"])
|
||||||
|
if o.get("custom_sql_joins"):
|
||||||
|
h["X-Custom-SQL-Join"] = "|".join(o["custom_sql_joins"])
|
||||||
|
if o.get("custom_sql_or"):
|
||||||
|
h["X-Custom-SQL-Or"] = " OR ".join(o["custom_sql_or"])
|
||||||
|
if o.get("search_columns"):
|
||||||
|
h["X-SearchCols"] = ",".join(o["search_columns"])
|
||||||
|
for col, sql in (o.get("advanced_sql") or {}).items():
|
||||||
|
h[f"X-AdvSQL-{col}"] = sql
|
||||||
|
|
||||||
|
vs = o.get("vector_search")
|
||||||
|
if vs:
|
||||||
|
h[f"X-Vector-Search-{vs['column']}"] = vs.get("metric") or "l2"
|
||||||
|
h["X-Vector-Search-Vector"] = json.dumps(vs["vector"], separators=(",", ":"))
|
||||||
|
if vs.get("as"):
|
||||||
|
h["X-Vector-Search-As"] = vs["as"]
|
||||||
|
if vs.get("direction"):
|
||||||
|
h["X-Vector-Search-Dir"] = vs["direction"]
|
||||||
|
|
||||||
|
for name, key in (
|
||||||
|
("X-Clean-JSON", "clean_json"),
|
||||||
|
("X-Distinct", "distinct"),
|
||||||
|
("X-SkipCount", "skip_count"),
|
||||||
|
("X-SkipCache", "skip_cache"),
|
||||||
|
("X-Transaction-Atomic", "atomic_transaction"),
|
||||||
|
("X-Single-Record-As-Object", "single_record_as_object"),
|
||||||
|
):
|
||||||
|
if o.get(key) is not None:
|
||||||
|
h[name] = _bool(o[key])
|
||||||
|
|
||||||
|
if o.get("pk_row"):
|
||||||
|
h["X-PKRow"] = o["pk_row"]
|
||||||
|
|
||||||
|
fmt = o.get("response_format")
|
||||||
|
if fmt:
|
||||||
|
h[{"simple": "X-SimpleApi", "detail": "X-DetailApi", "syncfusion": "X-Syncfusion"}[fmt]] = "true"
|
||||||
|
|
||||||
|
if o.get("xfiles"):
|
||||||
|
h["X-Files"] = encode_header_value(json.dumps(o["xfiles"], separators=(",", ":")))
|
||||||
|
|
||||||
|
if o.get("fetch_row_number"):
|
||||||
|
h["X-Fetch-RowNumber"] = o["fetch_row_number"]
|
||||||
|
|
||||||
|
for cc in o.get("computedColumns") or []:
|
||||||
|
h[f"X-CQL-SEL-{cc['name']}"] = cc["expression"]
|
||||||
|
|
||||||
|
if o.get("customOperators"):
|
||||||
|
h["X-Custom-SQL-W"] = " AND ".join(co["sql"] for co in o["customOperators"])
|
||||||
|
|
||||||
|
return h
|
||||||
|
|
||||||
|
|
||||||
|
def _int(s: Optional[str]) -> int:
|
||||||
|
try:
|
||||||
|
return int(s) # type: ignore[arg-type]
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
def _wrap(response: httpx.Response) -> APIResponse:
|
||||||
|
"""Wrap a raw restheadspec body, deriving metadata from Content-Range / X-Limit."""
|
||||||
|
data = parse_json(response)
|
||||||
|
if not response.is_success:
|
||||||
|
raise error_from(response, data)
|
||||||
|
cr = response.headers.get("content-range")
|
||||||
|
total = _int(cr.split("/")[-1]) if cr else 0
|
||||||
|
offset = _int(cr.split("/")[0].split("-")[0].split(" ")[-1]) if cr else 0
|
||||||
|
return {
|
||||||
|
"data": data,
|
||||||
|
"success": True,
|
||||||
|
"error": data.get("error") if isinstance(data, dict) else None,
|
||||||
|
"metadata": {
|
||||||
|
"count": total,
|
||||||
|
"total": total,
|
||||||
|
"filtered": total,
|
||||||
|
"offset": offset,
|
||||||
|
"limit": _int(response.headers.get("x-limit")),
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
class _Base:
|
||||||
|
def __init__(
|
||||||
|
self,
|
||||||
|
base_url: str,
|
||||||
|
token: Optional[str] = None,
|
||||||
|
headers: Optional[Mapping[str, str]] = None,
|
||||||
|
timeout: Optional[float] = 30.0,
|
||||||
|
):
|
||||||
|
self.base_url = base_url
|
||||||
|
self.token = token
|
||||||
|
self.headers = dict(headers or {}) # snapshot
|
||||||
|
self.timeout = timeout
|
||||||
|
|
||||||
|
def _base_headers(self) -> Dict[str, str]:
|
||||||
|
return client_headers(self.token, self.headers)
|
||||||
|
|
||||||
|
def _req(self, method, schema, entity, id, options=None, body=None):
|
||||||
|
opt = build_headers(options) if options else {}
|
||||||
|
return (
|
||||||
|
method,
|
||||||
|
build_url(self.base_url, schema, entity, id),
|
||||||
|
merge_headers(self._base_headers(), opt),
|
||||||
|
body,
|
||||||
|
)
|
||||||
|
|
||||||
|
def _read_req(self, schema, entity, id, options):
|
||||||
|
return self._req("GET", schema, entity, id, options)
|
||||||
|
|
||||||
|
def _create_req(self, schema, entity, data, options):
|
||||||
|
return self._req("POST", schema, entity, None, options, data)
|
||||||
|
|
||||||
|
def _update_req(self, schema, entity, id, data, options):
|
||||||
|
return self._req("PUT", schema, entity, id, options, data)
|
||||||
|
|
||||||
|
def _delete_req(self, schema, entity, id):
|
||||||
|
return self._req("DELETE", schema, entity, id)
|
||||||
|
|
||||||
|
|
||||||
|
class HeaderSpecClient(_Base):
|
||||||
|
"""Synchronous client. Use as a context manager or call close()."""
|
||||||
|
|
||||||
|
def __init__(self, *args: Any, transport: Optional[httpx.BaseTransport] = None, **kwargs: Any):
|
||||||
|
super().__init__(*args, **kwargs)
|
||||||
|
self._http = httpx.Client(timeout=self.timeout, transport=transport)
|
||||||
|
|
||||||
|
def close(self) -> None:
|
||||||
|
self._http.close()
|
||||||
|
|
||||||
|
def __enter__(self) -> "HeaderSpecClient":
|
||||||
|
return self
|
||||||
|
|
||||||
|
def __exit__(self, *exc: Any) -> None:
|
||||||
|
self.close()
|
||||||
|
|
||||||
|
def _send(self, req) -> APIResponse:
|
||||||
|
method, url, headers, body = req
|
||||||
|
return _wrap(self._http.request(method, url, headers=headers, json=body))
|
||||||
|
|
||||||
|
def read(self, schema: str, entity: str, id: Optional[str] = None, options: Optional[HeaderSpecOptions] = None) -> APIResponse:
|
||||||
|
return self._send(self._read_req(schema, entity, id, options))
|
||||||
|
|
||||||
|
def create(self, schema: str, entity: str, data: Any, options: Optional[HeaderSpecOptions] = None) -> APIResponse:
|
||||||
|
return self._send(self._create_req(schema, entity, data, options))
|
||||||
|
|
||||||
|
def update(self, schema: str, entity: str, id: str, data: Any, options: Optional[HeaderSpecOptions] = None) -> APIResponse:
|
||||||
|
return self._send(self._update_req(schema, entity, id, data, options))
|
||||||
|
|
||||||
|
def delete(self, schema: str, entity: str, id: str) -> APIResponse:
|
||||||
|
return self._send(self._delete_req(schema, entity, id))
|
||||||
|
|
||||||
|
|
||||||
|
class AsyncHeaderSpecClient(_Base):
|
||||||
|
"""Asyncio client. Use as an async context manager or await aclose()."""
|
||||||
|
|
||||||
|
def __init__(self, *args: Any, transport: Optional[httpx.AsyncBaseTransport] = None, **kwargs: Any):
|
||||||
|
super().__init__(*args, **kwargs)
|
||||||
|
self._http = httpx.AsyncClient(timeout=self.timeout, transport=transport)
|
||||||
|
|
||||||
|
async def aclose(self) -> None:
|
||||||
|
await self._http.aclose()
|
||||||
|
|
||||||
|
async def __aenter__(self) -> "AsyncHeaderSpecClient":
|
||||||
|
return self
|
||||||
|
|
||||||
|
async def __aexit__(self, *exc: Any) -> None:
|
||||||
|
await self.aclose()
|
||||||
|
|
||||||
|
async def _send(self, req) -> APIResponse:
|
||||||
|
method, url, headers, body = req
|
||||||
|
return _wrap(await self._http.request(method, url, headers=headers, json=body))
|
||||||
|
|
||||||
|
async def read(self, schema: str, entity: str, id: Optional[str] = None, options: Optional[HeaderSpecOptions] = None) -> APIResponse:
|
||||||
|
return await self._send(self._read_req(schema, entity, id, options))
|
||||||
|
|
||||||
|
async def create(self, schema: str, entity: str, data: Any, options: Optional[HeaderSpecOptions] = None) -> APIResponse:
|
||||||
|
return await self._send(self._create_req(schema, entity, data, options))
|
||||||
|
|
||||||
|
async def update(self, schema: str, entity: str, id: str, data: Any, options: Optional[HeaderSpecOptions] = None) -> APIResponse:
|
||||||
|
return await self._send(self._update_req(schema, entity, id, data, options))
|
||||||
|
|
||||||
|
async def delete(self, schema: str, entity: str, id: str) -> APIResponse:
|
||||||
|
return await self._send(self._delete_req(schema, entity, id))
|
||||||
@@ -0,0 +1,76 @@
|
|||||||
|
"""Shared HTTP helpers for the REST clients."""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from typing import Any, Dict, Mapping, Optional
|
||||||
|
from urllib.parse import quote
|
||||||
|
|
||||||
|
|
||||||
|
class ResolveSpecError(Exception):
|
||||||
|
"""Raised on a non-2xx response or an unsuccessful API result."""
|
||||||
|
|
||||||
|
def __init__(
|
||||||
|
self,
|
||||||
|
message: str,
|
||||||
|
status_code: Optional[int] = None,
|
||||||
|
code: Optional[str] = None,
|
||||||
|
details: Any = None,
|
||||||
|
detail: Optional[str] = None,
|
||||||
|
):
|
||||||
|
super().__init__(message)
|
||||||
|
self.message = message
|
||||||
|
self.status_code = status_code
|
||||||
|
self.code = code
|
||||||
|
self.details = details
|
||||||
|
self.detail = detail # server-side reason (funcspec / restheadspec errors)
|
||||||
|
|
||||||
|
|
||||||
|
def merge_headers(*sources: Mapping[str, str]) -> Dict[str, str]:
|
||||||
|
"""Merge HTTP headers case-insensitively; the last source wins and keeps its spelling."""
|
||||||
|
result: Dict[str, str] = {}
|
||||||
|
for source in sources:
|
||||||
|
for name, value in source.items():
|
||||||
|
for existing in [k for k in result if k.lower() == name.lower()]:
|
||||||
|
del result[existing]
|
||||||
|
result[name] = value
|
||||||
|
return result
|
||||||
|
|
||||||
|
|
||||||
|
def client_headers(token: Optional[str], headers: Optional[Mapping[str, str]]) -> Dict[str, str]:
|
||||||
|
"""Content-Type < custom headers < bearer token."""
|
||||||
|
return merge_headers(
|
||||||
|
{"Content-Type": "application/json"},
|
||||||
|
headers or {},
|
||||||
|
{"Authorization": f"Bearer {token}"} if token else {},
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def build_url(base_url: str, schema: str, entity: str, id: Optional[Any] = None) -> str:
|
||||||
|
url = f"{base_url.rstrip('/')}/{quote(schema, safe='')}/{quote(entity, safe='')}"
|
||||||
|
if id is not None and id != "":
|
||||||
|
url += f"/{quote(str(id), safe='')}"
|
||||||
|
return url
|
||||||
|
|
||||||
|
|
||||||
|
def drop_none(d: Mapping[str, Any]) -> Dict[str, Any]:
|
||||||
|
return {k: v for k, v in d.items() if v is not None}
|
||||||
|
|
||||||
|
|
||||||
|
def parse_json(response: Any) -> Any:
|
||||||
|
try:
|
||||||
|
return response.json()
|
||||||
|
except ValueError:
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def error_from(response: Any, data: Any) -> ResolveSpecError:
|
||||||
|
err = data.get("error") if isinstance(data, dict) else None
|
||||||
|
err = err if isinstance(err, dict) else {}
|
||||||
|
text = (response.text or "").strip() if data is None else ""
|
||||||
|
fallback = text[:200] or f"{response.reason_phrase} ({response.status_code})"
|
||||||
|
return ResolveSpecError(
|
||||||
|
err.get("message") or fallback,
|
||||||
|
status_code=response.status_code,
|
||||||
|
code=err.get("code"),
|
||||||
|
details=err.get("details"),
|
||||||
|
detail=err.get("detail"),
|
||||||
|
)
|
||||||
@@ -0,0 +1,137 @@
|
|||||||
|
"""ResolveSpec client: JSON body protocol (POST {operation, data, options})."""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from typing import Any, Dict, List, Mapping, Optional, Tuple
|
||||||
|
|
||||||
|
import httpx
|
||||||
|
|
||||||
|
from .http import build_url, client_headers, drop_none, error_from, parse_json
|
||||||
|
from .types import APIResponse, Options, RecordId
|
||||||
|
|
||||||
|
|
||||||
|
def _url_id(id: Optional[RecordId]) -> Optional[str]:
|
||||||
|
return str(id) if isinstance(id, (int, str)) else None
|
||||||
|
|
||||||
|
|
||||||
|
def _body_id(id: Optional[RecordId]) -> Optional[List[str]]:
|
||||||
|
return id if isinstance(id, list) else None
|
||||||
|
|
||||||
|
|
||||||
|
class _Base:
|
||||||
|
def __init__(
|
||||||
|
self,
|
||||||
|
base_url: str,
|
||||||
|
token: Optional[str] = None,
|
||||||
|
headers: Optional[Mapping[str, str]] = None,
|
||||||
|
timeout: Optional[float] = 30.0,
|
||||||
|
):
|
||||||
|
self.base_url = base_url
|
||||||
|
self.token = token
|
||||||
|
self.headers = dict(headers or {}) # snapshot
|
||||||
|
self.timeout = timeout
|
||||||
|
|
||||||
|
def _headers(self) -> Dict[str, str]:
|
||||||
|
return client_headers(self.token, self.headers)
|
||||||
|
|
||||||
|
def _request(
|
||||||
|
self, method: str, schema: str, entity: str, id: Optional[str], body: Optional[Dict[str, Any]]
|
||||||
|
) -> Tuple[str, str, Dict[str, str], Optional[Dict[str, Any]]]:
|
||||||
|
return method, build_url(self.base_url, schema, entity, id), self._headers(), body
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _result(response: httpx.Response) -> APIResponse:
|
||||||
|
data = parse_json(response)
|
||||||
|
if not response.is_success:
|
||||||
|
raise error_from(response, data)
|
||||||
|
return data
|
||||||
|
|
||||||
|
# request builders (shared by sync and async)
|
||||||
|
def _metadata_req(self, schema, entity):
|
||||||
|
return self._request("GET", schema, entity, None, None)
|
||||||
|
|
||||||
|
def _read_req(self, schema, entity, id, options):
|
||||||
|
body = drop_none({"operation": "read", "id": _body_id(id), "options": options})
|
||||||
|
return self._request("POST", schema, entity, _url_id(id), body)
|
||||||
|
|
||||||
|
def _create_req(self, schema, entity, data, options):
|
||||||
|
body = drop_none({"operation": "create", "data": data, "options": options})
|
||||||
|
return self._request("POST", schema, entity, None, body)
|
||||||
|
|
||||||
|
def _update_req(self, schema, entity, data, id, options):
|
||||||
|
body = drop_none({"operation": "update", "id": _body_id(id), "data": data, "options": options})
|
||||||
|
return self._request("POST", schema, entity, _url_id(id), body)
|
||||||
|
|
||||||
|
def _delete_req(self, schema, entity, id):
|
||||||
|
return self._request("POST", schema, entity, str(id), {"operation": "delete"})
|
||||||
|
|
||||||
|
|
||||||
|
class ResolveSpecClient(_Base):
|
||||||
|
"""Synchronous client. Use as a context manager or call close()."""
|
||||||
|
|
||||||
|
def __init__(self, *args: Any, transport: Optional[httpx.BaseTransport] = None, **kwargs: Any):
|
||||||
|
super().__init__(*args, **kwargs)
|
||||||
|
self._http = httpx.Client(timeout=self.timeout, transport=transport)
|
||||||
|
|
||||||
|
def close(self) -> None:
|
||||||
|
self._http.close()
|
||||||
|
|
||||||
|
def __enter__(self) -> "ResolveSpecClient":
|
||||||
|
return self
|
||||||
|
|
||||||
|
def __exit__(self, *exc: Any) -> None:
|
||||||
|
self.close()
|
||||||
|
|
||||||
|
def _send(self, req) -> APIResponse:
|
||||||
|
method, url, headers, body = req
|
||||||
|
return self._result(self._http.request(method, url, headers=headers, json=body))
|
||||||
|
|
||||||
|
def get_metadata(self, schema: str, entity: str) -> APIResponse:
|
||||||
|
return self._send(self._metadata_req(schema, entity))
|
||||||
|
|
||||||
|
def read(self, schema: str, entity: str, id: Optional[RecordId] = None, options: Optional[Options] = None) -> APIResponse:
|
||||||
|
return self._send(self._read_req(schema, entity, id, options))
|
||||||
|
|
||||||
|
def create(self, schema: str, entity: str, data: Any, options: Optional[Options] = None) -> APIResponse:
|
||||||
|
return self._send(self._create_req(schema, entity, data, options))
|
||||||
|
|
||||||
|
def update(self, schema: str, entity: str, data: Any, id: Optional[RecordId] = None, options: Optional[Options] = None) -> APIResponse:
|
||||||
|
return self._send(self._update_req(schema, entity, data, id, options))
|
||||||
|
|
||||||
|
def delete(self, schema: str, entity: str, id: Any) -> APIResponse:
|
||||||
|
return self._send(self._delete_req(schema, entity, id))
|
||||||
|
|
||||||
|
|
||||||
|
class AsyncResolveSpecClient(_Base):
|
||||||
|
"""Asyncio client. Use as an async context manager or await aclose()."""
|
||||||
|
|
||||||
|
def __init__(self, *args: Any, transport: Optional[httpx.AsyncBaseTransport] = None, **kwargs: Any):
|
||||||
|
super().__init__(*args, **kwargs)
|
||||||
|
self._http = httpx.AsyncClient(timeout=self.timeout, transport=transport)
|
||||||
|
|
||||||
|
async def aclose(self) -> None:
|
||||||
|
await self._http.aclose()
|
||||||
|
|
||||||
|
async def __aenter__(self) -> "AsyncResolveSpecClient":
|
||||||
|
return self
|
||||||
|
|
||||||
|
async def __aexit__(self, *exc: Any) -> None:
|
||||||
|
await self.aclose()
|
||||||
|
|
||||||
|
async def _send(self, req) -> APIResponse:
|
||||||
|
method, url, headers, body = req
|
||||||
|
return self._result(await self._http.request(method, url, headers=headers, json=body))
|
||||||
|
|
||||||
|
async def get_metadata(self, schema: str, entity: str) -> APIResponse:
|
||||||
|
return await self._send(self._metadata_req(schema, entity))
|
||||||
|
|
||||||
|
async def read(self, schema: str, entity: str, id: Optional[RecordId] = None, options: Optional[Options] = None) -> APIResponse:
|
||||||
|
return await self._send(self._read_req(schema, entity, id, options))
|
||||||
|
|
||||||
|
async def create(self, schema: str, entity: str, data: Any, options: Optional[Options] = None) -> APIResponse:
|
||||||
|
return await self._send(self._create_req(schema, entity, data, options))
|
||||||
|
|
||||||
|
async def update(self, schema: str, entity: str, data: Any, id: Optional[RecordId] = None, options: Optional[Options] = None) -> APIResponse:
|
||||||
|
return await self._send(self._update_req(schema, entity, data, id, options))
|
||||||
|
|
||||||
|
async def delete(self, schema: str, entity: str, id: Any) -> APIResponse:
|
||||||
|
return await self._send(self._delete_req(schema, entity, id))
|
||||||
@@ -0,0 +1,166 @@
|
|||||||
|
"""Types aligned with Go pkg/common/types.go. Dict keys are the wire names."""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from typing import Any, Dict, List, NotRequired, TypedDict, Union
|
||||||
|
|
||||||
|
Operator = str # eq neq gt gte lt lte like ilike in contains startswith endswith
|
||||||
|
# between between_inclusive is_null is_not_null
|
||||||
|
# st_dwithin bbox (spatial) | l2_within cosine_within ip_within (vector)
|
||||||
|
Operation = str # read | create | update | delete
|
||||||
|
SortDirection = str # asc | desc | ASC | DESC
|
||||||
|
VectorMetric = str # l2 | cosine | ip
|
||||||
|
ResponseFormat = str # simple | detail | syncfusion
|
||||||
|
|
||||||
|
RecordId = Union[int, str, List[str]]
|
||||||
|
|
||||||
|
|
||||||
|
class Parameter(TypedDict):
|
||||||
|
name: str
|
||||||
|
value: str
|
||||||
|
sequence: NotRequired[int]
|
||||||
|
|
||||||
|
|
||||||
|
class FilterOption(TypedDict):
|
||||||
|
column: str
|
||||||
|
operator: str
|
||||||
|
value: Any
|
||||||
|
logic_operator: NotRequired[str] # "AND" | "OR"
|
||||||
|
|
||||||
|
|
||||||
|
class SortOption(TypedDict):
|
||||||
|
column: str
|
||||||
|
direction: str
|
||||||
|
|
||||||
|
|
||||||
|
class CustomOperator(TypedDict):
|
||||||
|
name: str
|
||||||
|
sql: str
|
||||||
|
|
||||||
|
|
||||||
|
class ComputedColumn(TypedDict):
|
||||||
|
name: str
|
||||||
|
expression: str
|
||||||
|
|
||||||
|
|
||||||
|
class PreloadOption(TypedDict, total=False):
|
||||||
|
relation: str
|
||||||
|
table_name: str
|
||||||
|
columns: List[str]
|
||||||
|
omit_columns: List[str]
|
||||||
|
sort: List[SortOption]
|
||||||
|
filters: List[FilterOption]
|
||||||
|
where: str
|
||||||
|
limit: int
|
||||||
|
offset: int
|
||||||
|
updateable: bool
|
||||||
|
computed_ql: Dict[str, str]
|
||||||
|
recursive: bool
|
||||||
|
primary_key: str
|
||||||
|
related_key: str
|
||||||
|
foreign_key: str
|
||||||
|
recursive_child_key: str
|
||||||
|
sql_joins: List[str]
|
||||||
|
join_aliases: List[str]
|
||||||
|
|
||||||
|
|
||||||
|
# `as` is a keyword, so the functional syntax is required.
|
||||||
|
VectorSearchOption = TypedDict(
|
||||||
|
"VectorSearchOption",
|
||||||
|
{
|
||||||
|
"column": str,
|
||||||
|
"vector": List[float],
|
||||||
|
"metric": str, # l2 (default) | cosine | ip
|
||||||
|
"as": str, # distance column alias, default _distance
|
||||||
|
"direction": str, # asc (default) | desc
|
||||||
|
},
|
||||||
|
total=False,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class ExpandOption(TypedDict, total=False):
|
||||||
|
relation: str
|
||||||
|
columns: List[str]
|
||||||
|
|
||||||
|
|
||||||
|
class XFiles(TypedDict, total=False):
|
||||||
|
tablename: str
|
||||||
|
schema: str
|
||||||
|
primarykey: str
|
||||||
|
foreignkey: str
|
||||||
|
relatedkey: str
|
||||||
|
sort: List[str]
|
||||||
|
prefix: str
|
||||||
|
editable: bool
|
||||||
|
recursive: bool
|
||||||
|
expand: bool
|
||||||
|
rownumber: bool
|
||||||
|
skipcount: bool
|
||||||
|
offset: int
|
||||||
|
limit: int
|
||||||
|
columns: List[str]
|
||||||
|
omit_columns: List[str]
|
||||||
|
cql_columns: List[str]
|
||||||
|
sql_joins: List[str]
|
||||||
|
sql_or: List[str]
|
||||||
|
sql_and: List[str]
|
||||||
|
parenttables: List["XFiles"]
|
||||||
|
childtables: List["XFiles"]
|
||||||
|
filter_fields: List[Dict[str, str]]
|
||||||
|
cursor_forward: str
|
||||||
|
cursor_backward: str
|
||||||
|
|
||||||
|
|
||||||
|
class Options(TypedDict, total=False):
|
||||||
|
preload: List[PreloadOption]
|
||||||
|
columns: List[str]
|
||||||
|
omit_columns: List[str]
|
||||||
|
filters: List[FilterOption]
|
||||||
|
sort: List[SortOption]
|
||||||
|
limit: int
|
||||||
|
offset: int
|
||||||
|
customOperators: List[CustomOperator]
|
||||||
|
computedColumns: List[ComputedColumn]
|
||||||
|
parameters: List[Parameter]
|
||||||
|
cursor_forward: str
|
||||||
|
cursor_backward: str
|
||||||
|
fetch_row_number: str
|
||||||
|
vector_search: VectorSearchOption
|
||||||
|
|
||||||
|
|
||||||
|
class HeaderSpecOptions(Options, total=False):
|
||||||
|
"""Options only available to the header-based (restheadspec) protocol."""
|
||||||
|
|
||||||
|
expand: List[ExpandOption] # X-Expand
|
||||||
|
custom_sql_joins: List[str] # X-Custom-SQL-Join
|
||||||
|
custom_sql_or: List[str] # X-Custom-SQL-Or
|
||||||
|
search_columns: List[str] # X-SearchCols
|
||||||
|
advanced_sql: Dict[str, str] # X-AdvSQL-{col}
|
||||||
|
clean_json: bool # X-Clean-JSON
|
||||||
|
distinct: bool # X-Distinct
|
||||||
|
skip_count: bool # X-SkipCount
|
||||||
|
skip_cache: bool # X-SkipCache
|
||||||
|
pk_row: str # X-PKRow
|
||||||
|
response_format: str # X-SimpleApi / X-DetailApi / X-Syncfusion
|
||||||
|
single_record_as_object: bool # X-Single-Record-As-Object
|
||||||
|
atomic_transaction: bool # X-Transaction-Atomic
|
||||||
|
xfiles: XFiles # X-Files
|
||||||
|
|
||||||
|
|
||||||
|
class FuncSpecOptions(TypedDict, total=False):
|
||||||
|
"""Options understood by funcspec endpoints (sent as X-* headers)."""
|
||||||
|
|
||||||
|
filters: List[FilterOption] # eq+AND -> X-FieldFilter; others X-SearchOp / X-SearchOr (one per column)
|
||||||
|
search_filters: Dict[str, str] # X-SearchFilter-{col}: text ILIKE
|
||||||
|
custom_sql_where: str # X-Custom-SQL-W
|
||||||
|
custom_sql_or: str # X-Custom-SQL-Or
|
||||||
|
sort: List[SortOption] # sent as SQL ORDER BY terms ("col DESC")
|
||||||
|
limit: int
|
||||||
|
offset: int
|
||||||
|
distinct: bool
|
||||||
|
skip_count: bool
|
||||||
|
skip_cache: bool
|
||||||
|
response_format: str # simple | detail | syncfusion
|
||||||
|
|
||||||
|
|
||||||
|
# Responses are plain dicts: {"success", "data", "metadata"?, "error"?}
|
||||||
|
APIResponse = Dict[str, Any]
|
||||||
@@ -0,0 +1,335 @@
|
|||||||
|
"""WebSocketSpec client (asyncio). Mirrors the Go websocketspec message protocol."""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import asyncio
|
||||||
|
import json
|
||||||
|
import logging
|
||||||
|
import uuid
|
||||||
|
from dataclasses import dataclass, field
|
||||||
|
from typing import Any, Awaitable, Callable, Dict, List, Optional, Union
|
||||||
|
|
||||||
|
from websockets.asyncio.client import ClientConnection, connect
|
||||||
|
|
||||||
|
from .http import ResolveSpecError
|
||||||
|
from .types import FilterOption, PreloadOption, SortOption
|
||||||
|
|
||||||
|
log = logging.getLogger("resolvespec.websocket")
|
||||||
|
|
||||||
|
# Connection states
|
||||||
|
DISCONNECTED = "disconnected"
|
||||||
|
CONNECTING = "connecting"
|
||||||
|
CONNECTED = "connected"
|
||||||
|
DISCONNECTING = "disconnecting"
|
||||||
|
RECONNECTING = "reconnecting"
|
||||||
|
|
||||||
|
Notification = Dict[str, Any]
|
||||||
|
Callback = Callable[[Any], Union[None, Awaitable[None]]]
|
||||||
|
EVENTS = ("connect", "disconnect", "error", "message", "state_change")
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class Subscription:
|
||||||
|
id: str
|
||||||
|
entity: str
|
||||||
|
schema: Optional[str] = None
|
||||||
|
options: Optional[Dict[str, Any]] = None
|
||||||
|
callback: Optional[Callback] = field(default=None, repr=False)
|
||||||
|
|
||||||
|
|
||||||
|
def _drop_none(d: Dict[str, Any]) -> Dict[str, Any]:
|
||||||
|
return {k: v for k, v in d.items() if v is not None}
|
||||||
|
|
||||||
|
|
||||||
|
class WebSocketClient:
|
||||||
|
"""
|
||||||
|
Usage:
|
||||||
|
async with WebSocketClient("ws://localhost:8080/ws") as ws:
|
||||||
|
rows = await ws.read("users", schema="public", limit=10)
|
||||||
|
|
||||||
|
Events (`on(event, callback)`): connect, disconnect, error, message, state_change.
|
||||||
|
Callbacks may be sync or async.
|
||||||
|
"""
|
||||||
|
|
||||||
|
def __init__(
|
||||||
|
self,
|
||||||
|
url: str,
|
||||||
|
*,
|
||||||
|
reconnect: bool = True,
|
||||||
|
reconnect_interval: float = 3.0,
|
||||||
|
max_reconnect_attempts: int = 10,
|
||||||
|
heartbeat_interval: float = 30.0,
|
||||||
|
request_timeout: float = 30.0,
|
||||||
|
subscribe_timeout: float = 10.0,
|
||||||
|
headers: Optional[Dict[str, str]] = None,
|
||||||
|
):
|
||||||
|
self.url = url
|
||||||
|
self.reconnect = reconnect
|
||||||
|
self.reconnect_interval = reconnect_interval
|
||||||
|
self.max_reconnect_attempts = max_reconnect_attempts
|
||||||
|
self.heartbeat_interval = heartbeat_interval
|
||||||
|
self.request_timeout = request_timeout
|
||||||
|
self.subscribe_timeout = subscribe_timeout
|
||||||
|
self.headers = dict(headers or {})
|
||||||
|
|
||||||
|
self._ws: Optional[ClientConnection] = None
|
||||||
|
self._state = DISCONNECTED
|
||||||
|
self._pending: Dict[str, "asyncio.Future[Dict[str, Any]]"] = {}
|
||||||
|
self._subscriptions: Dict[str, Subscription] = {}
|
||||||
|
self._listeners: Dict[str, Callback] = {}
|
||||||
|
self._tasks: List["asyncio.Task[Any]"] = []
|
||||||
|
self._reader: Optional["asyncio.Task[Any]"] = None
|
||||||
|
self._manual_close = False
|
||||||
|
|
||||||
|
# ---- lifecycle -------------------------------------------------------
|
||||||
|
|
||||||
|
async def __aenter__(self) -> "WebSocketClient":
|
||||||
|
await self.connect()
|
||||||
|
return self
|
||||||
|
|
||||||
|
async def __aexit__(self, *exc: Any) -> None:
|
||||||
|
await self.close()
|
||||||
|
|
||||||
|
async def connect(self) -> None:
|
||||||
|
if self.is_connected():
|
||||||
|
return
|
||||||
|
self._manual_close = False
|
||||||
|
self._set_state(CONNECTING)
|
||||||
|
try:
|
||||||
|
self._ws = await connect(self.url, additional_headers=self.headers or None)
|
||||||
|
except Exception as e:
|
||||||
|
self._set_state(DISCONNECTED)
|
||||||
|
await self._emit("error", e)
|
||||||
|
raise
|
||||||
|
self._set_state(CONNECTED)
|
||||||
|
self._reader = asyncio.create_task(self._read_loop(self._ws))
|
||||||
|
self._heartbeat = asyncio.create_task(self._heartbeat_loop())
|
||||||
|
await self._emit("connect")
|
||||||
|
|
||||||
|
async def close(self) -> None:
|
||||||
|
self._manual_close = True
|
||||||
|
self._set_state(DISCONNECTING)
|
||||||
|
for t in (self._reader, getattr(self, "_heartbeat", None), getattr(self, "_reconnect_task", None)):
|
||||||
|
if t and t is not asyncio.current_task():
|
||||||
|
t.cancel()
|
||||||
|
if self._ws:
|
||||||
|
await self._ws.close()
|
||||||
|
self._ws = None
|
||||||
|
self._fail_pending(ResolveSpecError("WebSocket closed"))
|
||||||
|
self._set_state(DISCONNECTED)
|
||||||
|
|
||||||
|
def is_connected(self) -> bool:
|
||||||
|
return self._ws is not None and self._state == CONNECTED
|
||||||
|
|
||||||
|
@property
|
||||||
|
def state(self) -> str:
|
||||||
|
return self._state
|
||||||
|
|
||||||
|
def on(self, event: str, callback: Callback) -> None:
|
||||||
|
if event not in EVENTS:
|
||||||
|
raise ValueError(f"unknown event {event!r}; expected one of {EVENTS}")
|
||||||
|
self._listeners[event] = callback
|
||||||
|
|
||||||
|
def off(self, event: str) -> None:
|
||||||
|
self._listeners.pop(event, None)
|
||||||
|
|
||||||
|
def get_subscriptions(self) -> List[Subscription]:
|
||||||
|
return list(self._subscriptions.values())
|
||||||
|
|
||||||
|
# ---- operations ------------------------------------------------------
|
||||||
|
|
||||||
|
async def request(
|
||||||
|
self,
|
||||||
|
operation: str,
|
||||||
|
entity: str,
|
||||||
|
*,
|
||||||
|
schema: Optional[str] = None,
|
||||||
|
record_id: Optional[str] = None,
|
||||||
|
data: Any = None,
|
||||||
|
options: Optional[Dict[str, Any]] = None,
|
||||||
|
) -> Any:
|
||||||
|
message = _drop_none({
|
||||||
|
"type": "request",
|
||||||
|
"operation": operation,
|
||||||
|
"entity": entity,
|
||||||
|
"schema": schema,
|
||||||
|
"record_id": record_id,
|
||||||
|
"data": data,
|
||||||
|
"options": options,
|
||||||
|
})
|
||||||
|
response = await self._call(message, self.request_timeout, "Request")
|
||||||
|
return response.get("data")
|
||||||
|
|
||||||
|
async def read(
|
||||||
|
self,
|
||||||
|
entity: str,
|
||||||
|
*,
|
||||||
|
schema: Optional[str] = None,
|
||||||
|
record_id: Optional[str] = None,
|
||||||
|
filters: Optional[List[FilterOption]] = None,
|
||||||
|
columns: Optional[List[str]] = None,
|
||||||
|
sort: Optional[List[SortOption]] = None,
|
||||||
|
preload: Optional[List[PreloadOption]] = None,
|
||||||
|
limit: Optional[int] = None,
|
||||||
|
offset: Optional[int] = None,
|
||||||
|
) -> Any:
|
||||||
|
options = _drop_none({
|
||||||
|
"filters": filters, "columns": columns, "sort": sort,
|
||||||
|
"preload": preload, "limit": limit, "offset": offset,
|
||||||
|
})
|
||||||
|
return await self.request("read", entity, schema=schema, record_id=record_id, options=options)
|
||||||
|
|
||||||
|
async def create(self, entity: str, data: Any, *, schema: Optional[str] = None) -> Any:
|
||||||
|
return await self.request("create", entity, schema=schema, data=data)
|
||||||
|
|
||||||
|
async def update(self, entity: str, id: str, data: Any, *, schema: Optional[str] = None) -> Any:
|
||||||
|
return await self.request("update", entity, schema=schema, record_id=id, data=data)
|
||||||
|
|
||||||
|
async def delete(self, entity: str, id: str, *, schema: Optional[str] = None) -> None:
|
||||||
|
await self.request("delete", entity, schema=schema, record_id=id)
|
||||||
|
|
||||||
|
async def meta(self, entity: str, *, schema: Optional[str] = None) -> Any:
|
||||||
|
return await self.request("meta", entity, schema=schema)
|
||||||
|
|
||||||
|
async def subscribe(
|
||||||
|
self,
|
||||||
|
entity: str,
|
||||||
|
callback: Callback,
|
||||||
|
*,
|
||||||
|
schema: Optional[str] = None,
|
||||||
|
filters: Optional[List[FilterOption]] = None,
|
||||||
|
) -> str:
|
||||||
|
message = _drop_none({
|
||||||
|
"type": "subscription",
|
||||||
|
"operation": "subscribe",
|
||||||
|
"entity": entity,
|
||||||
|
"schema": schema,
|
||||||
|
"options": _drop_none({"filters": filters}),
|
||||||
|
})
|
||||||
|
response = await self._call(message, self.subscribe_timeout, "Subscription")
|
||||||
|
sub_id = (response.get("data") or {}).get("subscription_id")
|
||||||
|
if not sub_id:
|
||||||
|
raise ResolveSpecError("Subscription failed")
|
||||||
|
self._subscriptions[sub_id] = Subscription(
|
||||||
|
sub_id, entity, schema, _drop_none({"filters": filters}) or None, callback
|
||||||
|
)
|
||||||
|
return sub_id
|
||||||
|
|
||||||
|
async def unsubscribe(self, subscription_id: str) -> None:
|
||||||
|
message = {"type": "subscription", "operation": "unsubscribe", "subscription_id": subscription_id}
|
||||||
|
await self._call(message, self.subscribe_timeout, "Unsubscribe")
|
||||||
|
self._subscriptions.pop(subscription_id, None)
|
||||||
|
|
||||||
|
# ---- internals -------------------------------------------------------
|
||||||
|
|
||||||
|
async def _call(self, message: Dict[str, Any], timeout: float, what: str) -> Dict[str, Any]:
|
||||||
|
self._ensure_connected()
|
||||||
|
mid = str(uuid.uuid4())
|
||||||
|
message["id"] = mid
|
||||||
|
fut: "asyncio.Future[Dict[str, Any]]" = asyncio.get_running_loop().create_future()
|
||||||
|
self._pending[mid] = fut
|
||||||
|
try:
|
||||||
|
await self._ws.send(json.dumps(message)) # type: ignore[union-attr]
|
||||||
|
response = await asyncio.wait_for(fut, timeout)
|
||||||
|
except asyncio.TimeoutError:
|
||||||
|
raise ResolveSpecError(f"{what} timeout") from None
|
||||||
|
finally:
|
||||||
|
self._pending.pop(mid, None)
|
||||||
|
if not response.get("success"):
|
||||||
|
err = response.get("error") or {}
|
||||||
|
raise ResolveSpecError(
|
||||||
|
err.get("message") or f"{what} failed", code=err.get("code"), details=err.get("details")
|
||||||
|
)
|
||||||
|
return response
|
||||||
|
|
||||||
|
def _ensure_connected(self) -> None:
|
||||||
|
if not self.is_connected():
|
||||||
|
raise ResolveSpecError("WebSocket is not connected. Call connect() first.")
|
||||||
|
|
||||||
|
def _fail_pending(self, exc: Exception) -> None:
|
||||||
|
for fut in self._pending.values():
|
||||||
|
if not fut.done():
|
||||||
|
fut.set_exception(exc)
|
||||||
|
self._pending.clear()
|
||||||
|
|
||||||
|
async def _read_loop(self, ws: ClientConnection) -> None:
|
||||||
|
try:
|
||||||
|
async for raw in ws:
|
||||||
|
await self._handle_message(raw)
|
||||||
|
except asyncio.CancelledError:
|
||||||
|
raise
|
||||||
|
except Exception as e: # connection error
|
||||||
|
await self._emit("error", e)
|
||||||
|
# connection ended
|
||||||
|
if ws is not self._ws:
|
||||||
|
return
|
||||||
|
self._ws = None
|
||||||
|
if hb := getattr(self, "_heartbeat", None):
|
||||||
|
hb.cancel()
|
||||||
|
self._fail_pending(ResolveSpecError("WebSocket disconnected"))
|
||||||
|
self._set_state(DISCONNECTED)
|
||||||
|
await self._emit("disconnect", ws.close_code, ws.close_reason)
|
||||||
|
if self.reconnect and not self._manual_close:
|
||||||
|
self._reconnect_task = asyncio.create_task(self._reconnect())
|
||||||
|
|
||||||
|
async def _reconnect(self) -> None:
|
||||||
|
for attempt in range(1, self.max_reconnect_attempts + 1):
|
||||||
|
if self._manual_close:
|
||||||
|
return
|
||||||
|
log.debug("Reconnection attempt %d/%d", attempt, self.max_reconnect_attempts)
|
||||||
|
self._set_state(RECONNECTING)
|
||||||
|
await asyncio.sleep(self.reconnect_interval)
|
||||||
|
try:
|
||||||
|
await self.connect()
|
||||||
|
return
|
||||||
|
except Exception as e:
|
||||||
|
log.debug("Reconnection failed: %s", e)
|
||||||
|
self._set_state(DISCONNECTED)
|
||||||
|
|
||||||
|
async def _handle_message(self, raw: Union[str, bytes]) -> None:
|
||||||
|
try:
|
||||||
|
message = json.loads(raw)
|
||||||
|
except ValueError as e:
|
||||||
|
log.debug("Error parsing message: %s", e)
|
||||||
|
return
|
||||||
|
await self._emit("message", message)
|
||||||
|
kind = message.get("type")
|
||||||
|
if kind == "response":
|
||||||
|
fut = self._pending.get(message.get("id"))
|
||||||
|
if fut and not fut.done():
|
||||||
|
fut.set_result(message)
|
||||||
|
elif kind == "notification":
|
||||||
|
sub = self._subscriptions.get(message.get("subscription_id"))
|
||||||
|
if sub and sub.callback:
|
||||||
|
await _maybe_await(sub.callback(message))
|
||||||
|
elif kind != "pong":
|
||||||
|
log.debug("Unknown message type: %s", kind)
|
||||||
|
|
||||||
|
async def _heartbeat_loop(self) -> None:
|
||||||
|
try:
|
||||||
|
while True:
|
||||||
|
await asyncio.sleep(self.heartbeat_interval)
|
||||||
|
if self.is_connected():
|
||||||
|
await self._ws.send(json.dumps({"id": str(uuid.uuid4()), "type": "ping"})) # type: ignore[union-attr]
|
||||||
|
except asyncio.CancelledError:
|
||||||
|
raise
|
||||||
|
except Exception as e:
|
||||||
|
log.debug("Heartbeat failed: %s", e)
|
||||||
|
|
||||||
|
def _set_state(self, state: str) -> None:
|
||||||
|
if self._state != state:
|
||||||
|
self._state = state
|
||||||
|
cb = self._listeners.get("state_change")
|
||||||
|
if cb:
|
||||||
|
res = cb(state)
|
||||||
|
if asyncio.iscoroutine(res):
|
||||||
|
asyncio.ensure_future(res)
|
||||||
|
|
||||||
|
async def _emit(self, event: str, *args: Any) -> None:
|
||||||
|
cb = self._listeners.get(event)
|
||||||
|
if cb:
|
||||||
|
await _maybe_await(cb(*args))
|
||||||
|
|
||||||
|
|
||||||
|
async def _maybe_await(result: Any) -> None:
|
||||||
|
if asyncio.iscoroutine(result) or isinstance(result, asyncio.Future):
|
||||||
|
await result
|
||||||
@@ -0,0 +1,132 @@
|
|||||||
|
import httpx
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from resolvespec import AsyncFuncSpecClient, FuncSpecClient, ResolveSpecError
|
||||||
|
from resolvespec.funcspec import build_headers, build_query
|
||||||
|
from resolvespec.headerspec import decode_header_value
|
||||||
|
|
||||||
|
|
||||||
|
def make(handler, **kw):
|
||||||
|
return FuncSpecClient("http://localhost:3000", "tok", transport=httpx.MockTransport(handler), **kw)
|
||||||
|
|
||||||
|
|
||||||
|
def capture(status=200, body=None, headers=None):
|
||||||
|
seen = []
|
||||||
|
|
||||||
|
def handler(req):
|
||||||
|
seen.append(req)
|
||||||
|
return httpx.Response(status, json=body if body is not None else [], headers=headers)
|
||||||
|
|
||||||
|
return seen, handler
|
||||||
|
|
||||||
|
|
||||||
|
def test_filters():
|
||||||
|
h = build_headers({"filters": [
|
||||||
|
{"column": "status", "operator": "eq", "value": "active"},
|
||||||
|
{"column": "age", "operator": "gte", "value": 18},
|
||||||
|
{"column": "name", "operator": "contains", "value": "x", "logic_operator": "OR"},
|
||||||
|
{"column": "deleted", "operator": "is_null", "value": None},
|
||||||
|
{"column": "id", "operator": "in", "value": [1, 2]},
|
||||||
|
{"column": "p", "operator": "between_inclusive", "value": [1, 5]},
|
||||||
|
]})
|
||||||
|
assert h == {
|
||||||
|
"X-FieldFilter-status": "active",
|
||||||
|
"X-SearchOp-greaterthanorequal-age": "18",
|
||||||
|
"X-SearchOr-contains-name": "x",
|
||||||
|
"X-SearchOp-empty-deleted": "",
|
||||||
|
"X-SearchOp-in-id": "1,2",
|
||||||
|
"X-SearchOp-betweeninclusive-p": "1,5",
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def test_sort_is_sql_not_prefixed():
|
||||||
|
# server inserts sort verbatim into ORDER BY; "-col" would negate the column
|
||||||
|
h = build_headers({"sort": [{"column": "name", "direction": "asc"}, {"column": "created_at", "direction": "DESC"}]})
|
||||||
|
assert h["X-Sort"] == "name ASC,created_at DESC"
|
||||||
|
|
||||||
|
|
||||||
|
def test_misc_options():
|
||||||
|
h = build_headers({
|
||||||
|
"search_filters": {"name": "bob"}, "custom_sql_where": "a = 1", "custom_sql_or": "b = 2",
|
||||||
|
"limit": 5, "offset": 10, "distinct": True, "skip_count": True, "skip_cache": False,
|
||||||
|
"response_format": "syncfusion",
|
||||||
|
})
|
||||||
|
assert h == {
|
||||||
|
"X-SearchFilter-name": "bob", "X-Custom-SQL-W": "a = 1", "X-Custom-SQL-Or": "b = 2",
|
||||||
|
"X-Limit": "5", "X-Offset": "10", "X-Distinct": "true", "X-SkipCount": "true",
|
||||||
|
"X-SkipCache": "false", "X-Syncfusion": "true",
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def test_ambiguous_values_are_encoded():
|
||||||
|
h = build_headers({"custom_sql_where": "name = 'café'", "filters": [{"column": "c", "operator": "eq", "value": " pad "}]})
|
||||||
|
assert h["X-Custom-SQL-W"].startswith("ZIP_")
|
||||||
|
assert decode_header_value(h["X-Custom-SQL-W"]) == "name = 'café'"
|
||||||
|
assert decode_header_value(h["X-FieldFilter-c"]) == " pad "
|
||||||
|
|
||||||
|
|
||||||
|
def test_build_query():
|
||||||
|
q = build_query({"p-id": 5, "flag": True, "ids": [1, 2], "skip": None, "m": "match=ab"})
|
||||||
|
assert q == {"p-id": "5", "flag": "true", "ids": ["1", "2"], "m": "match=ab"}
|
||||||
|
|
||||||
|
|
||||||
|
def test_query_list_request_and_metadata():
|
||||||
|
seen, h = capture(206, [{"id": 1}, {"id": 2}], {"content-range": "items 10-12/50"})
|
||||||
|
with make(h) as c:
|
||||||
|
res = c.query_list("/api/orders", {"p-status": "open", "id": [1, 2]}, {"limit": 2, "offset": 10})
|
||||||
|
r = seen[0]
|
||||||
|
assert r.method == "GET"
|
||||||
|
assert r.url.path == "/api/orders"
|
||||||
|
assert r.url.params.multi_items() == [("p-status", "open"), ("id", "1"), ("id", "2")]
|
||||||
|
assert r.headers["x-limit"] == "2" and r.headers["authorization"] == "Bearer tok"
|
||||||
|
assert res == {
|
||||||
|
"success": True,
|
||||||
|
"data": [{"id": 1}, {"id": 2}],
|
||||||
|
"metadata": {"total": 50, "count": 2, "filtered": 50, "offset": 10, "limit": 2},
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def test_query_list_empty_result():
|
||||||
|
seen, h = capture(200, [], {"content-range": "items 0-0/0"})
|
||||||
|
with make(h) as c:
|
||||||
|
assert c.query_list("orders")["metadata"]["total"] == 0
|
||||||
|
assert seen[0].url.path == "/orders"
|
||||||
|
|
||||||
|
|
||||||
|
def test_query_single_has_no_metadata_and_method():
|
||||||
|
seen, h = capture(200, {"id": 1})
|
||||||
|
with make(h) as c:
|
||||||
|
res = c.query("api/order", method="post")
|
||||||
|
assert seen[0].method == "POST"
|
||||||
|
assert res == {"success": True, "data": {"id": 1}}
|
||||||
|
|
||||||
|
|
||||||
|
def test_detail_format_data_passthrough():
|
||||||
|
body = {"items": [{"a": 1}], "count": "1", "total": "1", "tablename": "/x", "tableprefix": "gsql"}
|
||||||
|
_, h = capture(200, body, {"content-range": "items 0-1/1"})
|
||||||
|
with make(h) as c:
|
||||||
|
assert c.query_list("x", options={"response_format": "detail"})["data"] == body
|
||||||
|
|
||||||
|
|
||||||
|
def test_server_error_shape():
|
||||||
|
err = {"success": False, "error": {"code": "query_failed", "message": "Failed to retrieve records", "detail": "no such column", "sql": "SELECT"}}
|
||||||
|
_, h = capture(400, err)
|
||||||
|
with make(h) as c:
|
||||||
|
with pytest.raises(ResolveSpecError, match="Failed to retrieve") as ei:
|
||||||
|
c.query_list("x")
|
||||||
|
assert ei.value.code == "query_failed" and ei.value.detail == "no such column" and ei.value.status_code == 400
|
||||||
|
|
||||||
|
|
||||||
|
def test_plain_text_panic_error():
|
||||||
|
with make(lambda r: httpx.Response(500, text="Internal server error: boom")) as c:
|
||||||
|
with pytest.raises(ResolveSpecError, match="boom"):
|
||||||
|
c.query("x")
|
||||||
|
|
||||||
|
|
||||||
|
async def test_async():
|
||||||
|
async def handler(req):
|
||||||
|
return httpx.Response(200, json=[{"id": 1}], headers={"content-range": "items 0-1/1"})
|
||||||
|
|
||||||
|
async with AsyncFuncSpecClient("http://localhost:3000", transport=httpx.MockTransport(handler)) as c:
|
||||||
|
assert (await c.query_list("x"))["metadata"]["total"] == 1
|
||||||
|
assert (await c.query("x"))["data"] == [{"id": 1}]
|
||||||
@@ -0,0 +1,236 @@
|
|||||||
|
import json
|
||||||
|
|
||||||
|
import httpx
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from resolvespec import (
|
||||||
|
AsyncHeaderSpecClient,
|
||||||
|
HeaderSpecClient,
|
||||||
|
ResolveSpecError,
|
||||||
|
build_headers,
|
||||||
|
decode_header_value,
|
||||||
|
encode_header_value,
|
||||||
|
get_headerspec_client,
|
||||||
|
)
|
||||||
|
import base64
|
||||||
|
|
||||||
|
CFG = dict(base_url="http://localhost:3000", token="tok")
|
||||||
|
|
||||||
|
|
||||||
|
# ---- build_headers (ported from headerspec.test.ts) ----
|
||||||
|
|
||||||
|
def test_preload_shared_where():
|
||||||
|
h = build_headers({"preload": [
|
||||||
|
{"relation": "Items", "columns": ["id"], "where": "active = true"},
|
||||||
|
{"relation": "Tags", "where": "active = true"},
|
||||||
|
]})
|
||||||
|
assert h["X-Preload"] == "Items:id|Tags"
|
||||||
|
assert h["X-Preload-Where"] == "active = true"
|
||||||
|
|
||||||
|
|
||||||
|
def test_preload_mixed_where_numbered():
|
||||||
|
h = build_headers({"preload": [
|
||||||
|
{"relation": "Items", "where": "a = 1"},
|
||||||
|
{"relation": "Category"},
|
||||||
|
{"relation": "Tags", "where": "b = 2"},
|
||||||
|
]})
|
||||||
|
assert h["X-Preload"] == "Category"
|
||||||
|
assert "X-Preload-Where" not in h
|
||||||
|
assert h["X-Preload-1"] == "Items" and h["X-Preload-1-Where"] == "a = 1"
|
||||||
|
assert h["X-Preload-2"] == "Tags" and h["X-Preload-2-Where"] == "b = 2"
|
||||||
|
|
||||||
|
|
||||||
|
def test_expand_joins_or_searchcols_advsql():
|
||||||
|
h = build_headers({
|
||||||
|
"expand": [{"relation": "Dept", "columns": ["id", "name"]}, {"relation": "Role"}],
|
||||||
|
"custom_sql_joins": ["LEFT JOIN a ON a.id = b.id", "INNER JOIN c ON c.id = b.cid"],
|
||||||
|
"custom_sql_or": ["x = 1", "y = 2"],
|
||||||
|
"search_columns": ["name", "email"],
|
||||||
|
"advanced_sql": {"total": "a + b"},
|
||||||
|
})
|
||||||
|
assert h["X-Expand"] == "Dept:id,name|Role"
|
||||||
|
assert h["X-Custom-SQL-Join"] == "LEFT JOIN a ON a.id = b.id|INNER JOIN c ON c.id = b.cid"
|
||||||
|
assert h["X-Custom-SQL-Or"] == "x = 1 OR y = 2"
|
||||||
|
assert h["X-SearchCols"] == "name,email"
|
||||||
|
assert h["X-AdvSQL-total"] == "a + b"
|
||||||
|
|
||||||
|
|
||||||
|
def test_flags_pkrow_format():
|
||||||
|
h = build_headers({
|
||||||
|
"clean_json": True, "distinct": True, "skip_count": True, "skip_cache": False,
|
||||||
|
"atomic_transaction": True, "single_record_as_object": False,
|
||||||
|
"pk_row": "42", "response_format": "detail",
|
||||||
|
})
|
||||||
|
assert h["X-Clean-JSON"] == "true"
|
||||||
|
assert h["X-Distinct"] == "true"
|
||||||
|
assert h["X-SkipCount"] == "true"
|
||||||
|
assert h["X-SkipCache"] == "false"
|
||||||
|
assert h["X-Transaction-Atomic"] == "true"
|
||||||
|
assert h["X-Single-Record-As-Object"] == "false"
|
||||||
|
assert h["X-PKRow"] == "42"
|
||||||
|
assert h["X-DetailApi"] == "true"
|
||||||
|
|
||||||
|
|
||||||
|
def test_spatial_and_vector_filters():
|
||||||
|
h = build_headers({"filters": [
|
||||||
|
{"column": "geom", "operator": "st_dwithin", "value": {"geom": "POINT(0 0)", "distance": 5}, "logic_operator": "OR"},
|
||||||
|
{"column": "emb", "operator": "cosine_within", "value": {"vector": [1, 2], "distance": 0.3}},
|
||||||
|
]})
|
||||||
|
assert json.loads(h["X-SpatialFilter-geom"]) == {
|
||||||
|
"op": "st_dwithin", "value": {"geom": "POINT(0 0)", "distance": 5}, "logic": "or"}
|
||||||
|
assert json.loads(h["X-VectorFilter-emb"])["op"] == "cosine_within"
|
||||||
|
|
||||||
|
|
||||||
|
def test_vector_search():
|
||||||
|
h = build_headers({"vector_search": {"column": "emb", "vector": [0.1, 0.2], "metric": "cosine", "as": "dist", "direction": "desc"}})
|
||||||
|
assert h["X-Vector-Search-emb"] == "cosine"
|
||||||
|
assert h["X-Vector-Search-Vector"] == "[0.1,0.2]"
|
||||||
|
assert h["X-Vector-Search-As"] == "dist"
|
||||||
|
assert h["X-Vector-Search-Dir"] == "desc"
|
||||||
|
|
||||||
|
|
||||||
|
def test_xfiles_zip():
|
||||||
|
xf = {"tablename": "users", "prefix": "USR", "limit": 10}
|
||||||
|
h = build_headers({"xfiles": xf})
|
||||||
|
assert h["X-Files"].startswith("ZIP_")
|
||||||
|
assert json.loads(decode_header_value(h["X-Files"])) == xf
|
||||||
|
|
||||||
|
|
||||||
|
def test_columns_and_omit():
|
||||||
|
assert build_headers({"columns": ["id", "name", "email"]})["X-Select-Fields"] == "id,name,email"
|
||||||
|
assert build_headers({"omit_columns": ["secret", "internal"]})["X-Not-Select-Fields"] == "secret,internal"
|
||||||
|
|
||||||
|
|
||||||
|
def test_filters():
|
||||||
|
assert build_headers({"filters": [{"column": "status", "operator": "eq", "value": "active"}]})["X-FieldFilter-status"] == "active"
|
||||||
|
assert build_headers({"filters": [{"column": "age", "operator": "gte", "value": 18}]})["X-SearchOp-greaterthanorequal-age"] == "18"
|
||||||
|
assert build_headers({"filters": [{"column": "name", "operator": "contains", "value": "test", "logic_operator": "OR"}]})["X-SearchOr-contains-name"] == "test"
|
||||||
|
assert build_headers({"filters": [{"column": "price", "operator": "between", "value": [10, 100]}]})["X-SearchOp-between-price"] == "10,100"
|
||||||
|
assert build_headers({"filters": [{"column": "deleted_at", "operator": "is_null", "value": None}]})["X-SearchOp-empty-deleted_at"] == ""
|
||||||
|
assert build_headers({"filters": [{"column": "id", "operator": "in", "value": [1, 2, 3]}]})["X-SearchOp-in-id"] == "1,2,3"
|
||||||
|
assert build_headers({"filters": [{"column": "a", "operator": "eq", "value": True}]})["X-FieldFilter-a"] == "true"
|
||||||
|
|
||||||
|
|
||||||
|
def test_sort_pagination_cursor():
|
||||||
|
h = build_headers({
|
||||||
|
"sort": [{"column": "name", "direction": "asc"}, {"column": "created_at", "direction": "DESC"}],
|
||||||
|
"limit": 25, "offset": 0, "cursor_forward": "abc", "cursor_backward": "xyz",
|
||||||
|
})
|
||||||
|
assert h["X-Sort"] == "+name,-created_at"
|
||||||
|
assert h["X-Limit"] == "25" and h["X-Offset"] == "0"
|
||||||
|
assert h["X-Cursor-Forward"] == "abc" and h["X-Cursor-Backward"] == "xyz"
|
||||||
|
|
||||||
|
|
||||||
|
def test_preload_basic_rownumber_computed_custom():
|
||||||
|
h = build_headers({
|
||||||
|
"preload": [{"relation": "Items", "columns": ["id", "name"]}, {"relation": "Category"}],
|
||||||
|
"fetch_row_number": "42",
|
||||||
|
"computedColumns": [{"name": "total", "expression": "price * qty"}],
|
||||||
|
"customOperators": [{"name": "a", "sql": "status = 'active'"}, {"name": "v", "sql": "verified = true"}],
|
||||||
|
})
|
||||||
|
assert h["X-Preload"] == "Items:id,name|Category"
|
||||||
|
assert h["X-Fetch-RowNumber"] == "42"
|
||||||
|
assert h["X-CQL-SEL-total"] == "price * qty"
|
||||||
|
assert h["X-Custom-SQL-W"] == "status = 'active' AND verified = true"
|
||||||
|
|
||||||
|
|
||||||
|
def test_empty_options():
|
||||||
|
assert build_headers({}) == {}
|
||||||
|
|
||||||
|
|
||||||
|
# ---- encode / decode ----
|
||||||
|
|
||||||
|
def test_roundtrip():
|
||||||
|
for s in ("some complex value with spaces & symbols!", "café ☕ 你好"):
|
||||||
|
enc = encode_header_value(s)
|
||||||
|
assert enc.startswith("ZIP_")
|
||||||
|
assert decode_header_value(enc) == s
|
||||||
|
|
||||||
|
|
||||||
|
def test_decode_double_underscore_and_plain():
|
||||||
|
assert decode_header_value("__" + base64.b64encode(b"hello").decode()) == "hello"
|
||||||
|
assert decode_header_value("__" + base64.b64encode("café ☕".encode()).decode()) == "café ☕"
|
||||||
|
assert decode_header_value("plain") == "plain"
|
||||||
|
|
||||||
|
|
||||||
|
def test_decode_nested():
|
||||||
|
assert decode_header_value(encode_header_value(encode_header_value("x"))) == "x"
|
||||||
|
|
||||||
|
|
||||||
|
# ---- client ----
|
||||||
|
|
||||||
|
def make(handler, cls=HeaderSpecClient, **kw):
|
||||||
|
return cls(**{**CFG, **kw}, transport=httpx.MockTransport(handler))
|
||||||
|
|
||||||
|
|
||||||
|
def test_read_sends_get_with_headers():
|
||||||
|
seen = []
|
||||||
|
|
||||||
|
def handler(req):
|
||||||
|
seen.append(req)
|
||||||
|
return httpx.Response(200, json=[{"id": 1}], headers={"content-range": "0-9/100", "x-limit": "10"})
|
||||||
|
|
||||||
|
with make(handler) as c:
|
||||||
|
res = c.read("public", "users", options={"columns": ["id", "name"], "limit": 10})
|
||||||
|
r = seen[0]
|
||||||
|
assert str(r.url) == "http://localhost:3000/public/users"
|
||||||
|
assert r.method == "GET"
|
||||||
|
assert r.headers["x-select-fields"] == "id,name"
|
||||||
|
assert r.headers["x-limit"] == "10"
|
||||||
|
assert r.headers["authorization"] == "Bearer tok"
|
||||||
|
assert res["success"] is True
|
||||||
|
assert res["data"] == [{"id": 1}]
|
||||||
|
assert res["metadata"] == {"count": 100, "total": 100, "filtered": 100, "offset": 0, "limit": 10}
|
||||||
|
|
||||||
|
|
||||||
|
def test_metadata_defaults_without_content_range():
|
||||||
|
with make(lambda r: httpx.Response(200, json=[])) as c:
|
||||||
|
assert c.read("public", "users")["metadata"]["total"] == 0
|
||||||
|
|
||||||
|
|
||||||
|
def test_read_with_id_create_update_delete():
|
||||||
|
seen = []
|
||||||
|
|
||||||
|
def handler(req):
|
||||||
|
seen.append(req)
|
||||||
|
return httpx.Response(200, json={})
|
||||||
|
|
||||||
|
with make(handler) as c:
|
||||||
|
c.read("public", "users", "42")
|
||||||
|
c.create("public", "users", {"name": "Test"})
|
||||||
|
c.update("public", "users", "1", {"name": "Updated"}, {"filters": [{"column": "active", "operator": "eq", "value": True}]})
|
||||||
|
c.delete("public", "users", "1")
|
||||||
|
assert str(seen[0].url) == "http://localhost:3000/public/users/42"
|
||||||
|
assert seen[1].method == "POST" and json.loads(seen[1].content) == {"name": "Test"}
|
||||||
|
assert seen[2].method == "PUT" and str(seen[2].url).endswith("/public/users/1")
|
||||||
|
assert seen[2].headers["x-fieldfilter-active"] == "true"
|
||||||
|
assert seen[3].method == "DELETE"
|
||||||
|
|
||||||
|
|
||||||
|
def test_error_response():
|
||||||
|
with make(lambda r: httpx.Response(400, json={"error": {"code": "err", "message": "fail"}})) as c:
|
||||||
|
with pytest.raises(ResolveSpecError, match="fail") as ei:
|
||||||
|
c.read("public", "users")
|
||||||
|
assert ei.value.status_code == 400 and ei.value.code == "err"
|
||||||
|
|
||||||
|
|
||||||
|
def test_error_non_json():
|
||||||
|
with make(lambda r: httpx.Response(502, text="bad gateway")) as c:
|
||||||
|
with pytest.raises(ResolveSpecError, match="bad gateway") as ei:
|
||||||
|
c.read("public", "users")
|
||||||
|
assert ei.value.status_code == 502
|
||||||
|
|
||||||
|
|
||||||
|
async def test_async_client():
|
||||||
|
async def handler(req):
|
||||||
|
return httpx.Response(200, json=[{"id": 1}])
|
||||||
|
|
||||||
|
async with AsyncHeaderSpecClient(**CFG, transport=httpx.MockTransport(handler)) as c:
|
||||||
|
res = await c.read("public", "users", options={"limit": 1})
|
||||||
|
assert res["data"] == [{"id": 1}]
|
||||||
|
|
||||||
|
|
||||||
|
def test_singleton():
|
||||||
|
a = get_headerspec_client("http://hs-singleton:3000")
|
||||||
|
assert a is get_headerspec_client("http://hs-singleton:3000")
|
||||||
|
assert a is not get_headerspec_client("http://hs-singleton-b:3000")
|
||||||
@@ -0,0 +1,210 @@
|
|||||||
|
import json
|
||||||
|
|
||||||
|
import httpx
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from resolvespec import (
|
||||||
|
AsyncHeaderSpecClient,
|
||||||
|
AsyncResolveSpecClient,
|
||||||
|
HeaderSpecClient,
|
||||||
|
ResolveSpecClient,
|
||||||
|
ResolveSpecError,
|
||||||
|
get_headerspec_client,
|
||||||
|
get_resolvespec_client,
|
||||||
|
)
|
||||||
|
|
||||||
|
CFG = dict(base_url="http://localhost:3000", token="test-token")
|
||||||
|
|
||||||
|
|
||||||
|
def make(handler, **kw):
|
||||||
|
return ResolveSpecClient(**{**CFG, **kw}, transport=httpx.MockTransport(handler))
|
||||||
|
|
||||||
|
|
||||||
|
def ok(_req):
|
||||||
|
return httpx.Response(200, json={"success": True, "data": [{"id": 1}]})
|
||||||
|
|
||||||
|
|
||||||
|
def capture():
|
||||||
|
seen = []
|
||||||
|
|
||||||
|
def handler(req):
|
||||||
|
seen.append(req)
|
||||||
|
return httpx.Response(200, json={"success": True, "data": {"id": 1, "name": "Test"}})
|
||||||
|
|
||||||
|
return seen, handler
|
||||||
|
|
||||||
|
|
||||||
|
def body(req):
|
||||||
|
return json.loads(req.content)
|
||||||
|
|
||||||
|
|
||||||
|
def test_read_with_numeric_id():
|
||||||
|
seen, h = capture()
|
||||||
|
with make(h) as c:
|
||||||
|
assert c.read("public", "users", 1)["success"] is True
|
||||||
|
r = seen[0]
|
||||||
|
assert str(r.url) == "http://localhost:3000/public/users/1"
|
||||||
|
assert r.method == "POST"
|
||||||
|
assert r.headers["authorization"] == "Bearer test-token"
|
||||||
|
assert r.headers["content-type"] == "application/json"
|
||||||
|
assert body(r) == {"operation": "read"}
|
||||||
|
|
||||||
|
|
||||||
|
def test_read_array_id_goes_in_body():
|
||||||
|
seen, h = capture()
|
||||||
|
with make(h) as c:
|
||||||
|
c.read("public", "users", ["1", "2"])
|
||||||
|
assert str(seen[0].url) == "http://localhost:3000/public/users"
|
||||||
|
assert body(seen[0])["id"] == ["1", "2"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_read_options_passthrough():
|
||||||
|
seen, h = capture()
|
||||||
|
opts = {
|
||||||
|
"columns": ["id", "name"], "omit_columns": ["secret"],
|
||||||
|
"filters": [{"column": "active", "operator": "eq", "value": True}],
|
||||||
|
"sort": [{"column": "name", "direction": "asc"}],
|
||||||
|
"limit": 10, "offset": 0, "cursor_forward": "cursor1", "fetch_row_number": "5",
|
||||||
|
"customOperators": [{"name": "x", "sql": "a = 1"}],
|
||||||
|
}
|
||||||
|
with make(h) as c:
|
||||||
|
c.read("public", "users", options=opts)
|
||||||
|
assert body(seen[0])["options"] == opts
|
||||||
|
|
||||||
|
|
||||||
|
def test_create():
|
||||||
|
seen, h = capture()
|
||||||
|
with make(h) as c:
|
||||||
|
res = c.create("public", "users", {"name": "Test"})
|
||||||
|
assert res["data"]["name"] == "Test"
|
||||||
|
assert body(seen[0]) == {"operation": "create", "data": {"name": "Test"}}
|
||||||
|
|
||||||
|
|
||||||
|
def test_create_batch():
|
||||||
|
seen, h = capture()
|
||||||
|
with make(h) as c:
|
||||||
|
c.create("public", "users", [{"a": 1}, {"a": 2}])
|
||||||
|
assert body(seen[0])["data"] == [{"a": 1}, {"a": 2}]
|
||||||
|
|
||||||
|
|
||||||
|
def test_update_with_id_in_url_and_array():
|
||||||
|
seen, h = capture()
|
||||||
|
with make(h) as c:
|
||||||
|
c.update("public", "users", {"name": "X"}, 5)
|
||||||
|
c.update("public", "users", {"name": "X"}, ["1", "2"])
|
||||||
|
assert str(seen[0].url).endswith("/public/users/5")
|
||||||
|
assert body(seen[0]) == {"operation": "update", "data": {"name": "X"}}
|
||||||
|
assert str(seen[1].url).endswith("/public/users")
|
||||||
|
assert body(seen[1])["id"] == ["1", "2"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_update_preserves_empty_string_and_null():
|
||||||
|
seen, h = capture()
|
||||||
|
with make(h) as c:
|
||||||
|
c.update("public", "users", {"a": "", "b": None}, 1)
|
||||||
|
assert body(seen[0])["data"] == {"a": "", "b": None}
|
||||||
|
|
||||||
|
|
||||||
|
def test_delete():
|
||||||
|
seen, h = capture()
|
||||||
|
with make(h) as c:
|
||||||
|
c.delete("public", "users", 1)
|
||||||
|
assert str(seen[0].url).endswith("/public/users/1")
|
||||||
|
assert body(seen[0]) == {"operation": "delete"}
|
||||||
|
|
||||||
|
|
||||||
|
def test_get_metadata():
|
||||||
|
seen, h = capture()
|
||||||
|
with make(h) as c:
|
||||||
|
c.get_metadata("public", "users")
|
||||||
|
assert seen[0].method == "GET"
|
||||||
|
assert str(seen[0].url) == "http://localhost:3000/public/users"
|
||||||
|
assert not seen[0].content
|
||||||
|
|
||||||
|
|
||||||
|
def test_error_uses_server_message():
|
||||||
|
with make(lambda r: httpx.Response(404, json={"success": False, "error": {"code": "not_found", "message": "nope"}})) as c:
|
||||||
|
with pytest.raises(ResolveSpecError, match="nope") as ei:
|
||||||
|
c.read("public", "users", 1)
|
||||||
|
assert ei.value.status_code == 404 and ei.value.code == "not_found"
|
||||||
|
|
||||||
|
|
||||||
|
def test_id_is_url_quoted():
|
||||||
|
seen, h = capture()
|
||||||
|
with make(h) as c:
|
||||||
|
c.read("public", "users", "a/b")
|
||||||
|
assert str(seen[0].url).endswith("/public/users/a%2Fb")
|
||||||
|
|
||||||
|
|
||||||
|
def test_trailing_slash_base_url():
|
||||||
|
seen, h = capture()
|
||||||
|
with make(h, base_url="http://localhost:3000/") as c:
|
||||||
|
c.read("public", "users")
|
||||||
|
assert str(seen[0].url) == "http://localhost:3000/public/users"
|
||||||
|
|
||||||
|
|
||||||
|
async def test_async_client():
|
||||||
|
async def handler(req):
|
||||||
|
return httpx.Response(200, json={"success": True, "data": [1]})
|
||||||
|
|
||||||
|
async with AsyncResolveSpecClient(**CFG, transport=httpx.MockTransport(handler)) as c:
|
||||||
|
assert (await c.read("public", "users"))["data"] == [1]
|
||||||
|
assert (await c.create("public", "users", {}))["success"]
|
||||||
|
assert (await c.update("public", "users", {}, 1))["success"]
|
||||||
|
assert (await c.delete("public", "users", 1))["success"]
|
||||||
|
assert (await c.get_metadata("public", "users"))["success"]
|
||||||
|
|
||||||
|
|
||||||
|
# ---- custom headers (ported from custom-headers.test.ts) ----
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("cls", [ResolveSpecClient, HeaderSpecClient])
|
||||||
|
def test_custom_headers_on_every_op_case_insensitive(cls):
|
||||||
|
seen = []
|
||||||
|
|
||||||
|
def handler(req):
|
||||||
|
seen.append(req)
|
||||||
|
return httpx.Response(200, json={"success": True, "data": []})
|
||||||
|
|
||||||
|
headers = {"X-Tenant": "acme", "authorization": "Basic ignored",
|
||||||
|
"content-type": "application/custom+json", "x-limit": "99"}
|
||||||
|
with cls("http://localhost:3000", "tok", headers, transport=httpx.MockTransport(handler)) as c:
|
||||||
|
c.read("public", "users", options={"limit": 10})
|
||||||
|
c.create("public", "users", {})
|
||||||
|
if cls is ResolveSpecClient:
|
||||||
|
c.update("public", "users", {}, "1")
|
||||||
|
c.get_metadata("public", "users")
|
||||||
|
else:
|
||||||
|
c.update("public", "users", "1", {})
|
||||||
|
c.delete("public", "users", "1")
|
||||||
|
for r in seen:
|
||||||
|
assert r.headers["x-tenant"] == "acme"
|
||||||
|
assert r.headers["authorization"] == "Bearer tok"
|
||||||
|
assert r.headers["content-type"] == "application/custom+json"
|
||||||
|
if cls is HeaderSpecClient:
|
||||||
|
assert seen[0].headers["x-limit"] == "10"
|
||||||
|
assert headers["authorization"] == "Basic ignored"
|
||||||
|
assert headers["x-limit"] == "99"
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("cls", [ResolveSpecClient, HeaderSpecClient])
|
||||||
|
def test_custom_auth_without_token(cls):
|
||||||
|
seen = []
|
||||||
|
|
||||||
|
def handler(req):
|
||||||
|
seen.append(req)
|
||||||
|
return httpx.Response(200, json={"success": True, "data": []})
|
||||||
|
|
||||||
|
with cls("http://localhost:3000", headers={"Authorization": "Basic custom"}, transport=httpx.MockTransport(handler)) as c:
|
||||||
|
c.read("public", "users")
|
||||||
|
assert seen[0].headers["authorization"] == "Basic custom"
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("factory", [get_resolvespec_client, get_headerspec_client])
|
||||||
|
def test_cache_isolation_and_snapshot(factory):
|
||||||
|
headers = {"X-Tenant": "acme", "X-App": "grid"}
|
||||||
|
first = factory("http://tenant-cache", "one", headers)
|
||||||
|
assert factory("http://tenant-cache", "one", {"x-app": "grid", "x-tenant": "acme"}) is first
|
||||||
|
assert factory("http://tenant-cache", "two", headers) is not first
|
||||||
|
headers["X-Tenant"] = "other"
|
||||||
|
assert factory("http://tenant-cache", "one", headers) is not first
|
||||||
|
assert first.headers["X-Tenant"] == "acme"
|
||||||
@@ -0,0 +1,152 @@
|
|||||||
|
import asyncio
|
||||||
|
import json
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
from websockets.asyncio.server import serve
|
||||||
|
|
||||||
|
from resolvespec import ResolveSpecError, WebSocketClient
|
||||||
|
|
||||||
|
|
||||||
|
class Server:
|
||||||
|
"""Minimal in-process WebSocketSpec server."""
|
||||||
|
|
||||||
|
def __init__(self):
|
||||||
|
self.received = []
|
||||||
|
self.conns = set()
|
||||||
|
self.respond = True
|
||||||
|
|
||||||
|
async def handler(self, ws):
|
||||||
|
self.conns.add(ws)
|
||||||
|
try:
|
||||||
|
async for raw in ws:
|
||||||
|
msg = json.loads(raw)
|
||||||
|
self.received.append(msg)
|
||||||
|
if msg["type"] == "ping":
|
||||||
|
await ws.send(json.dumps({"type": "pong"}))
|
||||||
|
continue
|
||||||
|
if not self.respond:
|
||||||
|
continue
|
||||||
|
await ws.send(json.dumps(self.reply(msg)))
|
||||||
|
finally:
|
||||||
|
self.conns.discard(ws)
|
||||||
|
|
||||||
|
def reply(self, msg):
|
||||||
|
base = {"id": msg["id"], "type": "response", "success": True, "timestamp": "t"}
|
||||||
|
if msg["type"] == "subscription" and msg["operation"] == "subscribe":
|
||||||
|
return {**base, "data": {"subscription_id": "sub-1"}}
|
||||||
|
if msg.get("entity") == "fail":
|
||||||
|
return {**base, "success": False, "error": {"code": "bad", "message": "boom"}}
|
||||||
|
return {**base, "data": {"echo": msg.get("operation"), "record_id": msg.get("record_id")}}
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
async def server():
|
||||||
|
s = Server()
|
||||||
|
async with serve(s.handler, "127.0.0.1", 0) as srv:
|
||||||
|
s.url = "ws://127.0.0.1:%d" % srv.sockets[0].getsockname()[1]
|
||||||
|
yield s
|
||||||
|
|
||||||
|
|
||||||
|
async def test_operations_and_message_shape(server):
|
||||||
|
async with WebSocketClient(server.url, reconnect=False) as c:
|
||||||
|
assert c.state == "connected"
|
||||||
|
assert await c.read("users", schema="public", record_id="1", limit=5, filters=[{"column": "a", "operator": "eq", "value": 1}]) == {"echo": "read", "record_id": "1"}
|
||||||
|
await c.create("users", {"n": 1}, schema="public")
|
||||||
|
await c.update("users", "2", {"n": 2})
|
||||||
|
await c.delete("users", "3")
|
||||||
|
await c.meta("users")
|
||||||
|
m = server.received
|
||||||
|
assert m[0]["type"] == "request" and m[0]["operation"] == "read"
|
||||||
|
assert m[0]["schema"] == "public" and m[0]["record_id"] == "1"
|
||||||
|
assert m[0]["options"] == {"filters": [{"column": "a", "operator": "eq", "value": 1}], "limit": 5}
|
||||||
|
assert m[1]["data"] == {"n": 1}
|
||||||
|
assert m[2]["record_id"] == "2"
|
||||||
|
assert [x["operation"] for x in m] == ["read", "create", "update", "delete", "meta"]
|
||||||
|
assert "schema" not in m[2]
|
||||||
|
assert len({x["id"] for x in m}) == 5
|
||||||
|
|
||||||
|
|
||||||
|
async def test_error_response_raises(server):
|
||||||
|
async with WebSocketClient(server.url, reconnect=False) as c:
|
||||||
|
with pytest.raises(ResolveSpecError, match="boom") as ei:
|
||||||
|
await c.read("fail")
|
||||||
|
assert ei.value.code == "bad"
|
||||||
|
|
||||||
|
|
||||||
|
async def test_request_timeout(server):
|
||||||
|
server.respond = False
|
||||||
|
async with WebSocketClient(server.url, reconnect=False, request_timeout=0.1) as c:
|
||||||
|
with pytest.raises(ResolveSpecError, match="timeout"):
|
||||||
|
await c.read("users")
|
||||||
|
assert not c._pending
|
||||||
|
|
||||||
|
|
||||||
|
async def test_not_connected_raises():
|
||||||
|
c = WebSocketClient("ws://127.0.0.1:1")
|
||||||
|
with pytest.raises(ResolveSpecError, match="not connected"):
|
||||||
|
await c.read("users")
|
||||||
|
|
||||||
|
|
||||||
|
async def test_subscribe_notify_unsubscribe(server):
|
||||||
|
got = asyncio.Queue()
|
||||||
|
async with WebSocketClient(server.url, reconnect=False) as c:
|
||||||
|
sid = await c.subscribe("users", got.put, schema="public", filters=[{"column": "a", "operator": "eq", "value": 1}])
|
||||||
|
assert sid == "sub-1"
|
||||||
|
assert [s.id for s in c.get_subscriptions()] == ["sub-1"]
|
||||||
|
assert server.received[0]["operation"] == "subscribe"
|
||||||
|
assert server.received[0]["options"] == {"filters": [{"column": "a", "operator": "eq", "value": 1}]}
|
||||||
|
for ws in server.conns:
|
||||||
|
await ws.send(json.dumps({"type": "notification", "operation": "create", "subscription_id": "sub-1",
|
||||||
|
"entity": "users", "data": {"id": 9}, "timestamp": "t"}))
|
||||||
|
n = await asyncio.wait_for(got.get(), 2)
|
||||||
|
assert n["data"] == {"id": 9}
|
||||||
|
await c.unsubscribe("sub-1")
|
||||||
|
assert c.get_subscriptions() == []
|
||||||
|
assert server.received[-1] == {**server.received[-1], "operation": "unsubscribe", "subscription_id": "sub-1"}
|
||||||
|
|
||||||
|
|
||||||
|
async def test_events_and_heartbeat(server):
|
||||||
|
events = []
|
||||||
|
c = WebSocketClient(server.url, reconnect=False, heartbeat_interval=0.05)
|
||||||
|
c.on("connect", lambda: events.append("connect"))
|
||||||
|
c.on("state_change", lambda s: events.append(s))
|
||||||
|
c.on("message", lambda m: events.append(("msg", m["type"])))
|
||||||
|
await c.connect()
|
||||||
|
await asyncio.sleep(0.2)
|
||||||
|
await c.close()
|
||||||
|
assert events[:3] == ["connecting", "connected", "connect"]
|
||||||
|
assert ("msg", "pong") in events
|
||||||
|
assert events[-1] == "disconnected"
|
||||||
|
assert any(m["type"] == "ping" for m in server.received)
|
||||||
|
with pytest.raises(ValueError):
|
||||||
|
c.on("bogus", lambda: None)
|
||||||
|
|
||||||
|
|
||||||
|
async def test_reconnect_after_server_drop(server):
|
||||||
|
states = []
|
||||||
|
c = WebSocketClient(server.url, reconnect=True, reconnect_interval=0.05)
|
||||||
|
c.on("state_change", states.append)
|
||||||
|
await c.connect()
|
||||||
|
for ws in list(server.conns):
|
||||||
|
await ws.close()
|
||||||
|
for _ in range(100):
|
||||||
|
if states.count("connected") >= 2:
|
||||||
|
break
|
||||||
|
await asyncio.sleep(0.05)
|
||||||
|
assert "reconnecting" in states
|
||||||
|
assert c.is_connected()
|
||||||
|
assert (await c.read("users"))["echo"] == "read"
|
||||||
|
await c.close()
|
||||||
|
|
||||||
|
|
||||||
|
async def test_pending_requests_fail_on_disconnect(server):
|
||||||
|
server.respond = False
|
||||||
|
c = WebSocketClient(server.url, reconnect=False)
|
||||||
|
await c.connect()
|
||||||
|
task = asyncio.create_task(c.read("users"))
|
||||||
|
await asyncio.sleep(0.05)
|
||||||
|
for ws in list(server.conns):
|
||||||
|
await ws.close()
|
||||||
|
with pytest.raises(ResolveSpecError, match="disconnected"):
|
||||||
|
await asyncio.wait_for(task, 2)
|
||||||
|
await c.close()
|
||||||
@@ -4,34 +4,34 @@
|
|||||||
|
|
||||||
### 1. ResolveSpec Client API
|
### 1. ResolveSpec Client API
|
||||||
|
|
||||||
- [ ] Core API implementation (read, create, update, delete, get_metadata)
|
- [x] Core API implementation (read, create, update, delete, get_metadata)
|
||||||
- [ ] Unit tests for API functions
|
- [x] Unit tests for API functions
|
||||||
- [ ] Integration tests with server
|
- [ ] Integration tests with server
|
||||||
- [ ] Error handling and edge cases
|
- [x] Error handling and edge cases
|
||||||
|
|
||||||
### 2. HeaderSpec Client API
|
### 2. HeaderSpec Client API
|
||||||
|
|
||||||
- [ ] Client API implementation
|
- [x] Client API implementation
|
||||||
- [ ] Unit tests
|
- [x] Unit tests
|
||||||
- [ ] Integration tests with server
|
- [ ] Integration tests with server
|
||||||
|
|
||||||
### 3. FunctionSpec Client API
|
### 3. FunctionSpec Client API
|
||||||
|
|
||||||
- [ ] Client API implementation
|
- [x] Client API implementation
|
||||||
- [ ] Unit tests
|
- [x] Unit tests
|
||||||
- [ ] Integration tests with server
|
- [ ] Integration tests with server
|
||||||
|
|
||||||
### 4. WebSocketSpec Client API
|
### 4. WebSocketSpec Client API
|
||||||
|
|
||||||
- [ ] WebSocketClient class implementation (read, create, update, delete, meta, subscribe, unsubscribe)
|
- [x] WebSocketClient class implementation (read, create, update, delete, meta, subscribe, unsubscribe)
|
||||||
- [ ] Unit tests for WebSocketClient
|
- [x] Unit tests for WebSocketClient
|
||||||
- [ ] Connection handling tests
|
- [x] Connection handling tests
|
||||||
- [ ] Subscription tests
|
- [x] Subscription tests
|
||||||
- [ ] Integration tests with server
|
- [ ] Integration tests with server
|
||||||
|
|
||||||
### 5. Testing Infrastructure
|
### 5. Testing Infrastructure
|
||||||
|
|
||||||
- [ ] Set up test framework (pytest)
|
- [x] Set up test framework (pytest)
|
||||||
- [ ] Configure test coverage reporting (pytest-cov)
|
- [ ] Configure test coverage reporting (pytest-cov)
|
||||||
- [ ] Add test utilities and fixtures
|
- [ ] Add test utilities and fixtures
|
||||||
- [ ] Create test documentation
|
- [ ] Create test documentation
|
||||||
@@ -43,8 +43,8 @@
|
|||||||
- [ ] Usage examples for each client API
|
- [ ] Usage examples for each client API
|
||||||
- [ ] Installation guide
|
- [ ] Installation guide
|
||||||
- [ ] Contributing guidelines
|
- [ ] Contributing guidelines
|
||||||
- [ ] README with quick start
|
- [x] README (cheatsheet)
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
**Last Updated:** 2026-02-07
|
**Last Updated:** 2026-09-30
|
||||||
@@ -0,0 +1,2 @@
|
|||||||
|
target/
|
||||||
|
Cargo.lock
|
||||||
@@ -0,0 +1,18 @@
|
|||||||
|
[package]
|
||||||
|
name = "resolvespec"
|
||||||
|
version = "0.1.0"
|
||||||
|
edition = "2021"
|
||||||
|
rust-version = "1.80"
|
||||||
|
description = "Client for ResolveSpec (JSON body) and FunctionSpec endpoints"
|
||||||
|
license = "MIT"
|
||||||
|
|
||||||
|
[dependencies]
|
||||||
|
reqwest = { version = "0.12", default-features = false, features = ["json", "rustls-tls"] }
|
||||||
|
serde = { version = "1", features = ["derive"] }
|
||||||
|
serde_json = "1"
|
||||||
|
base64 = "0.22"
|
||||||
|
thiserror = "1"
|
||||||
|
|
||||||
|
[dev-dependencies]
|
||||||
|
tokio = { version = "1", features = ["macros", "rt-multi-thread"] }
|
||||||
|
wiremock = "0.6"
|
||||||
@@ -0,0 +1,40 @@
|
|||||||
|
# resolvespec (Rust)
|
||||||
|
|
||||||
|
Rust client for ResolveSpec (JSON body) and FunctionSpec. Async (`reqwest` + `tokio`). MSRV 1.80.
|
||||||
|
|
||||||
|
## Clients
|
||||||
|
|
||||||
|
| Type | Constructor | Methods |
|
||||||
|
|---|---|---|
|
||||||
|
| `ResolveSpecClient` | `new(base_url)` / `from_builder(ClientBuilder)` | `get_metadata` `read` `create` `update` `delete` |
|
||||||
|
| `FuncSpecClient` | `new(base_url)` / `from_builder(ClientBuilder)` | `query` `query_list` `request` |
|
||||||
|
|
||||||
|
`ClientBuilder::new(url).token().header().timeout().http_client()`. Precedence: Content-Type < custom headers < bearer token.
|
||||||
|
|
||||||
|
## ResolveSpec
|
||||||
|
|
||||||
|
- `RecordId`: `Int`/`Str` → URL, `Many(Vec<String>)` → body (`From` impls provided).
|
||||||
|
- `Options` (`Default` + struct update), optional fields are `Option`/empty `Vec`.
|
||||||
|
- Result: `Response{success, data: serde_json::Value, metadata}`; `resp.decode::<T>()`.
|
||||||
|
|
||||||
|
## FunctionSpec
|
||||||
|
|
||||||
|
- Routes are server-defined: pass the `path`.
|
||||||
|
- `Params = BTreeMap<String, Param>` → query string (`Param::List` → repeated keys).
|
||||||
|
- `FuncSpecOptions` → `X-*` headers: `filters`, `search_filters`, `custom_sql_where`, `custom_sql_or`, `sort`, `limit`, `offset`, `distinct`, `skip_count`, `skip_cache`, `response_format`.
|
||||||
|
- `query_list` fills `metadata` from `Content-Range`; 206 is success.
|
||||||
|
|
||||||
|
## Server quirks
|
||||||
|
|
||||||
|
- `sort` is raw SQL in ORDER BY (client sends `col ASC|DESC`).
|
||||||
|
- One search operator per column.
|
||||||
|
- Values starting `ZIP_` / `__` are base64-decoded by the server.
|
||||||
|
- Non-ASCII, control chars and edge spaces are auto-encoded (`ZIP_`).
|
||||||
|
|
||||||
|
## Errors
|
||||||
|
|
||||||
|
`Error::Api { status, message, error: ApiError{code, message, detail, sql} }`, `Error::Http`, `Error::Json`.
|
||||||
|
|
||||||
|
## Test
|
||||||
|
|
||||||
|
`cargo test`
|
||||||
@@ -0,0 +1,90 @@
|
|||||||
|
use std::collections::HashMap;
|
||||||
|
use std::time::Duration;
|
||||||
|
|
||||||
|
use reqwest::header::{HeaderMap, HeaderName, HeaderValue, AUTHORIZATION, CONTENT_TYPE};
|
||||||
|
|
||||||
|
use crate::error::Result;
|
||||||
|
|
||||||
|
/// Shared HTTP configuration.
|
||||||
|
#[derive(Clone)]
|
||||||
|
pub(crate) struct Config {
|
||||||
|
pub base_url: String,
|
||||||
|
pub token: Option<String>,
|
||||||
|
pub headers: HashMap<String, String>,
|
||||||
|
pub http: reqwest::Client,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Builder options shared by both clients.
|
||||||
|
#[derive(Default, Clone)]
|
||||||
|
pub struct ClientBuilder {
|
||||||
|
base_url: String,
|
||||||
|
token: Option<String>,
|
||||||
|
headers: HashMap<String, String>,
|
||||||
|
timeout: Option<Duration>,
|
||||||
|
http: Option<reqwest::Client>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl ClientBuilder {
|
||||||
|
pub fn new(base_url: &str) -> Self {
|
||||||
|
Self { base_url: base_url.trim_end_matches('/').into(), timeout: Some(Duration::from_secs(30)), ..Default::default() }
|
||||||
|
}
|
||||||
|
pub fn token(mut self, token: &str) -> Self {
|
||||||
|
self.token = Some(token.into());
|
||||||
|
self
|
||||||
|
}
|
||||||
|
pub fn header(mut self, name: &str, value: &str) -> Self {
|
||||||
|
self.headers.insert(name.into(), value.into());
|
||||||
|
self
|
||||||
|
}
|
||||||
|
pub fn timeout(mut self, t: Duration) -> Self {
|
||||||
|
self.timeout = Some(t);
|
||||||
|
self
|
||||||
|
}
|
||||||
|
pub fn http_client(mut self, c: reqwest::Client) -> Self {
|
||||||
|
self.http = Some(c);
|
||||||
|
self
|
||||||
|
}
|
||||||
|
pub(crate) fn config(self) -> Result<Config> {
|
||||||
|
let http = match self.http {
|
||||||
|
Some(c) => c,
|
||||||
|
None => {
|
||||||
|
let mut b = reqwest::Client::builder();
|
||||||
|
if let Some(t) = self.timeout {
|
||||||
|
b = b.timeout(t);
|
||||||
|
}
|
||||||
|
b.build()?
|
||||||
|
}
|
||||||
|
};
|
||||||
|
Ok(Config { base_url: self.base_url, token: self.token, headers: self.headers, http })
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Config {
|
||||||
|
/// Content-Type < custom headers < extra (per-call) < bearer token.
|
||||||
|
pub fn headers(&self, extra: &HashMap<String, String>) -> HeaderMap {
|
||||||
|
let mut m = HeaderMap::new();
|
||||||
|
m.insert(CONTENT_TYPE, HeaderValue::from_static("application/json"));
|
||||||
|
for (k, v) in self.headers.iter().chain(extra.iter()) {
|
||||||
|
if let (Ok(n), Ok(v)) = (HeaderName::try_from(k.as_str()), HeaderValue::from_str(v)) {
|
||||||
|
m.insert(n, v);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if let Some(t) = &self.token {
|
||||||
|
if let Ok(v) = HeaderValue::from_str(&format!("Bearer {t}")) {
|
||||||
|
m.insert(AUTHORIZATION, v);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
m
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn path_segment(s: &str) -> String {
|
||||||
|
let mut out = String::new();
|
||||||
|
for b in s.bytes() {
|
||||||
|
match b {
|
||||||
|
b'A'..=b'Z' | b'a'..=b'z' | b'0'..=b'9' | b'-' | b'.' | b'_' | b'~' => out.push(b as char),
|
||||||
|
_ => out.push_str(&format!("%{b:02X}")),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
out
|
||||||
|
}
|
||||||
@@ -0,0 +1,34 @@
|
|||||||
|
use crate::types::ApiError;
|
||||||
|
|
||||||
|
/// Returned on transport failure, a non-2xx response or an unsuccessful API result.
|
||||||
|
#[derive(Debug, thiserror::Error)]
|
||||||
|
pub enum Error {
|
||||||
|
#[error("{message}")]
|
||||||
|
Api { status: u16, message: String, error: ApiError },
|
||||||
|
#[error(transparent)]
|
||||||
|
Http(#[from] reqwest::Error),
|
||||||
|
#[error(transparent)]
|
||||||
|
Json(#[from] serde_json::Error),
|
||||||
|
}
|
||||||
|
|
||||||
|
pub type Result<T> = std::result::Result<T, Error>;
|
||||||
|
|
||||||
|
pub(crate) fn error_from(status: u16, body: &str) -> Error {
|
||||||
|
let parsed: Option<serde_json::Value> = serde_json::from_str(body).ok();
|
||||||
|
let err: ApiError = parsed
|
||||||
|
.as_ref()
|
||||||
|
.and_then(|v| v.get("error"))
|
||||||
|
.and_then(|e| serde_json::from_value(e.clone()).ok())
|
||||||
|
.unwrap_or_default();
|
||||||
|
let message = if !err.message.is_empty() {
|
||||||
|
err.message.clone()
|
||||||
|
} else {
|
||||||
|
let text = if parsed.is_none() { body.trim().chars().take(200).collect::<String>() } else { String::new() };
|
||||||
|
if text.is_empty() {
|
||||||
|
format!("{} ({})", reqwest::StatusCode::from_u16(status).ok().and_then(|s| s.canonical_reason()).unwrap_or("Error"), status)
|
||||||
|
} else {
|
||||||
|
text
|
||||||
|
}
|
||||||
|
};
|
||||||
|
Error::Api { status, message, error: err }
|
||||||
|
}
|
||||||
@@ -0,0 +1,263 @@
|
|||||||
|
use std::collections::{BTreeMap, HashMap};
|
||||||
|
|
||||||
|
use base64::{engine::general_purpose::STANDARD, Engine};
|
||||||
|
use reqwest::Method;
|
||||||
|
use serde_json::Value;
|
||||||
|
|
||||||
|
use crate::client::{ClientBuilder, Config};
|
||||||
|
use crate::error::{error_from, Result};
|
||||||
|
use crate::types::{FilterOption, Metadata, Response, SortOption};
|
||||||
|
|
||||||
|
/// Options sent to funcspec endpoints as `X-*` headers.
|
||||||
|
///
|
||||||
|
/// Server behaviour (`pkg/funcspec`): `sort` is inserted raw into ORDER BY (so it is sent as SQL
|
||||||
|
/// terms); only one search operator per column is kept; values starting with `ZIP_` or `__`
|
||||||
|
/// are base64-decoded by the server, so such plaintext values cannot be sent faithfully.
|
||||||
|
#[derive(Debug, Clone, Default)]
|
||||||
|
pub struct FuncSpecOptions {
|
||||||
|
/// eq+AND -> X-FieldFilter; others X-SearchOp / X-SearchOr.
|
||||||
|
pub filters: Vec<FilterOption>,
|
||||||
|
/// X-SearchFilter-{col}: text ILIKE.
|
||||||
|
pub search_filters: BTreeMap<String, String>,
|
||||||
|
pub custom_sql_where: Option<String>,
|
||||||
|
pub custom_sql_or: Option<String>,
|
||||||
|
pub sort: Vec<SortOption>,
|
||||||
|
pub limit: Option<i64>,
|
||||||
|
pub offset: Option<i64>,
|
||||||
|
pub distinct: Option<bool>,
|
||||||
|
pub skip_count: Option<bool>,
|
||||||
|
pub skip_cache: Option<bool>,
|
||||||
|
/// simple | detail | syncfusion
|
||||||
|
pub response_format: Option<String>,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Query-string parameter value. `List` is sent as repeated keys (server: IN filter).
|
||||||
|
#[derive(Debug, Clone)]
|
||||||
|
pub enum Param {
|
||||||
|
Str(String),
|
||||||
|
Int(i64),
|
||||||
|
Bool(bool),
|
||||||
|
List(Vec<String>),
|
||||||
|
}
|
||||||
|
|
||||||
|
impl From<&str> for Param {
|
||||||
|
fn from(v: &str) -> Self {
|
||||||
|
Self::Str(v.into())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
impl From<String> for Param {
|
||||||
|
fn from(v: String) -> Self {
|
||||||
|
Self::Str(v)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
impl From<i64> for Param {
|
||||||
|
fn from(v: i64) -> Self {
|
||||||
|
Self::Int(v)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
impl From<bool> for Param {
|
||||||
|
fn from(v: bool) -> Self {
|
||||||
|
Self::Bool(v)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
impl From<Vec<String>> for Param {
|
||||||
|
fn from(v: Vec<String>) -> Self {
|
||||||
|
Self::List(v)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub type Params = BTreeMap<String, Param>;
|
||||||
|
|
||||||
|
fn operator(op: &str) -> &str {
|
||||||
|
match op {
|
||||||
|
"eq" => "equals",
|
||||||
|
"neq" => "notequals",
|
||||||
|
"gt" => "greaterthan",
|
||||||
|
"gte" => "greaterthanorequal",
|
||||||
|
"lt" => "lessthan",
|
||||||
|
"lte" => "lessthanorequal",
|
||||||
|
"like" | "ilike" | "contains" => "contains",
|
||||||
|
"startswith" => "beginswith",
|
||||||
|
"endswith" => "endswith",
|
||||||
|
"in" => "in",
|
||||||
|
"between" => "between",
|
||||||
|
"between_inclusive" => "betweeninclusive",
|
||||||
|
"is_null" => "empty",
|
||||||
|
"is_not_null" => "notempty",
|
||||||
|
other => other,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn scalar(v: &Value) -> String {
|
||||||
|
match v {
|
||||||
|
Value::Null => String::new(),
|
||||||
|
Value::String(s) => s.clone(),
|
||||||
|
Value::Array(a) => a.iter().map(scalar).collect::<Vec<_>>().join(","),
|
||||||
|
other => other.to_string(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Base64 (UTF-8) with the `ZIP_` prefix.
|
||||||
|
pub fn encode_header_value(v: &str) -> String {
|
||||||
|
format!("ZIP_{}", STANDARD.encode(v.as_bytes()))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Decode a value that may carry a `ZIP_` or `__` prefix (nested allowed).
|
||||||
|
pub fn decode_header_value(v: &str) -> String {
|
||||||
|
for p in ["ZIP_", "__"] {
|
||||||
|
if let Some(rest) = v.strip_prefix(p) {
|
||||||
|
let mut b64: String = rest.chars().filter(|c| !matches!(c, '\n' | '\r' | ' ')).collect();
|
||||||
|
while b64.len() % 4 != 0 {
|
||||||
|
b64.push('=');
|
||||||
|
}
|
||||||
|
return match STANDARD.decode(b64).ok().and_then(|b| String::from_utf8(b).ok()) {
|
||||||
|
Some(s) => decode_header_value(&s),
|
||||||
|
None => v.to_string(),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
}
|
||||||
|
v.to_string()
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Encode values that are unsafe as raw header/query text (non-ASCII, control chars, edge spaces).
|
||||||
|
fn safe(v: &str) -> String {
|
||||||
|
if v != v.trim() || v.chars().any(|c| !c.is_ascii() || c.is_ascii_control()) {
|
||||||
|
encode_header_value(v)
|
||||||
|
} else {
|
||||||
|
v.to_string()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Build the `X-*` headers understood by `funcspec.ParseParameters`.
|
||||||
|
pub fn build_headers(o: &FuncSpecOptions) -> BTreeMap<String, String> {
|
||||||
|
let mut h = BTreeMap::new();
|
||||||
|
for f in &o.filters {
|
||||||
|
let logic = f.logic_operator.as_deref().unwrap_or("AND");
|
||||||
|
let v = safe(&scalar(&f.value));
|
||||||
|
if f.operator == "eq" && logic == "AND" {
|
||||||
|
h.insert(format!("X-FieldFilter-{}", f.column), v);
|
||||||
|
} else {
|
||||||
|
let kind = if logic == "OR" { "X-SearchOr" } else { "X-SearchOp" };
|
||||||
|
h.insert(format!("{kind}-{}-{}", operator(&f.operator), f.column), v);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for (col, text) in &o.search_filters {
|
||||||
|
h.insert(format!("X-SearchFilter-{col}"), safe(text));
|
||||||
|
}
|
||||||
|
if let Some(v) = o.custom_sql_where.as_deref().filter(|s| !s.is_empty()) {
|
||||||
|
h.insert("X-Custom-SQL-W".into(), safe(v));
|
||||||
|
}
|
||||||
|
if let Some(v) = o.custom_sql_or.as_deref().filter(|s| !s.is_empty()) {
|
||||||
|
h.insert("X-Custom-SQL-Or".into(), safe(v));
|
||||||
|
}
|
||||||
|
if !o.sort.is_empty() {
|
||||||
|
let terms: Vec<String> = o
|
||||||
|
.sort
|
||||||
|
.iter()
|
||||||
|
.map(|s| format!("{} {}", s.column, if s.direction.eq_ignore_ascii_case("desc") { "DESC" } else { "ASC" }))
|
||||||
|
.collect();
|
||||||
|
h.insert("X-Sort".into(), safe(&terms.join(","))); // funcspec puts this verbatim into ORDER BY
|
||||||
|
}
|
||||||
|
if let Some(n) = o.limit {
|
||||||
|
h.insert("X-Limit".into(), n.to_string());
|
||||||
|
}
|
||||||
|
if let Some(n) = o.offset {
|
||||||
|
h.insert("X-Offset".into(), n.to_string());
|
||||||
|
}
|
||||||
|
for (name, v) in [("X-Distinct", o.distinct), ("X-SkipCount", o.skip_count), ("X-SkipCache", o.skip_cache)] {
|
||||||
|
if let Some(b) = v {
|
||||||
|
h.insert(name.into(), b.to_string());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
match o.response_format.as_deref() {
|
||||||
|
Some("simple") => h.insert("X-SimpleApi".into(), "true".into()),
|
||||||
|
Some("detail") => h.insert("X-DetailApi".into(), "true".into()),
|
||||||
|
Some("syncfusion") => h.insert("X-Syncfusion".into(), "true".into()),
|
||||||
|
_ => None,
|
||||||
|
};
|
||||||
|
h
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Build query-string pairs: bools -> true/false, lists -> repeated keys.
|
||||||
|
pub fn build_query(p: &Params) -> Vec<(String, String)> {
|
||||||
|
let mut out = Vec::new();
|
||||||
|
for (k, v) in p {
|
||||||
|
match v {
|
||||||
|
Param::Str(s) => out.push((k.clone(), safe(s))),
|
||||||
|
Param::Int(n) => out.push((k.clone(), n.to_string())),
|
||||||
|
Param::Bool(b) => out.push((k.clone(), b.to_string())),
|
||||||
|
Param::List(l) => out.extend(l.iter().map(|s| (k.clone(), safe(s)))),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
out
|
||||||
|
}
|
||||||
|
|
||||||
|
fn metadata(content_range: Option<&str>, limit: Option<i64>) -> Metadata {
|
||||||
|
let mut m = Metadata { limit: limit.unwrap_or(0), ..Default::default() };
|
||||||
|
if let Some(cr) = content_range {
|
||||||
|
// "items {start}-{end}/{total}"
|
||||||
|
let rest = cr.rsplit(' ').next().unwrap_or("");
|
||||||
|
if let Some((range, total)) = rest.split_once('/') {
|
||||||
|
if let (Some((s, e)), Ok(t)) = (range.split_once('-'), total.parse::<i64>()) {
|
||||||
|
if let (Ok(s), Ok(e)) = (s.parse::<i64>(), e.parse::<i64>()) {
|
||||||
|
m.total = t;
|
||||||
|
m.filtered = t;
|
||||||
|
m.count = e - s;
|
||||||
|
m.offset = s;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
m
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Client for user-defined SQL endpoints. Routes are defined by the server application.
|
||||||
|
#[derive(Clone)]
|
||||||
|
pub struct FuncSpecClient {
|
||||||
|
cfg: Config,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl FuncSpecClient {
|
||||||
|
pub fn new(base_url: &str) -> Result<Self> {
|
||||||
|
Self::from_builder(ClientBuilder::new(base_url))
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn from_builder(b: ClientBuilder) -> Result<Self> {
|
||||||
|
Ok(Self { cfg: b.config()? })
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn call(&self, method: Method, path: &str, params: &Params, options: Option<&FuncSpecOptions>, list: bool) -> Result<Response> {
|
||||||
|
let url = format!("{}/{}", self.cfg.base_url, path.trim_start_matches('/'));
|
||||||
|
let extra: HashMap<String, String> = options.map(|o| build_headers(o).into_iter().collect()).unwrap_or_default();
|
||||||
|
let resp = self.cfg.http.request(method, url).headers(self.cfg.headers(&extra)).query(&build_query(params)).send().await?;
|
||||||
|
let status = resp.status();
|
||||||
|
let cr = resp.headers().get("content-range").and_then(|v| v.to_str().ok()).map(str::to_owned);
|
||||||
|
let text = resp.text().await?;
|
||||||
|
if !status.is_success() {
|
||||||
|
// 206 Partial Content is success
|
||||||
|
return Err(error_from(status.as_u16(), &text));
|
||||||
|
}
|
||||||
|
let data = if text.trim().is_empty() { Value::Null } else { serde_json::from_str(&text)? };
|
||||||
|
Ok(Response {
|
||||||
|
success: true,
|
||||||
|
data,
|
||||||
|
metadata: list.then(|| metadata(cr.as_deref(), options.and_then(|o| o.limit))),
|
||||||
|
error: None,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Single-record endpoint (`SqlQuery`). `data` is the row object.
|
||||||
|
pub async fn query(&self, path: &str, params: &Params, options: Option<&FuncSpecOptions>) -> Result<Response> {
|
||||||
|
self.call(Method::GET, path, params, options, false).await
|
||||||
|
}
|
||||||
|
|
||||||
|
/// List endpoint (`SqlQueryList`). Metadata comes from Content-Range.
|
||||||
|
pub async fn query_list(&self, path: &str, params: &Params, options: Option<&FuncSpecOptions>) -> Result<Response> {
|
||||||
|
self.call(Method::GET, path, params, options, true).await
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Like `query` / `query_list` with an explicit HTTP method (routes are app-defined).
|
||||||
|
pub async fn request(&self, method: Method, path: &str, params: &Params, options: Option<&FuncSpecOptions>, list: bool) -> Result<Response> {
|
||||||
|
self.call(method, path, params, options, list).await
|
||||||
|
}
|
||||||
|
}
|
||||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user