Compare commits

...
Author SHA1 Message Date
Hermes AgentandClaude Sonnet 5.5 d36806047b feat(cli): add --watch mode to convert (#39)
Poll source files/directories and regenerate output on change.
Output path is excluded from watching; errors don't stop the loop.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
2026-10-03 10:29:42 +02:00
warkanum b38f53c603 docs(tests): add test coverage plans 2026-10-03 10:01:48 +02:00
warkanum ccba53c494 test: add podman/docker dbtest tool for postgres, mssql and mysql 2026-10-03 10:01:48 +02:00
sgcommand 53327b9a5a feat(ui): TUI indexes, views, sequences, scripts, domain/table assignment (#40) (#43)
Co-authored-by: SG Command <sgcommand@warky.dev>
2026-10-03 07:27:25 +00:00
sgcommand 734b14d48d docs: add usage examples for each format combination (#42)
Co-authored-by: SG Command <sgcommand@warky.dev>
2026-10-03 07:27:03 +00:00
sgcommand 938f0ed51f feat(cli): add --dry-run to convert, merge and split (#41)
Co-authored-by: SG Command <sgcommand@warky.dev>
2026-10-03 07:26:54 +00:00
warkanum 6e2e7eb19e feat(release): add rerelease target to move latest tag 2026-10-02 23:41:42 +02:00
warkanum 82e86cdf39 fix(release): run lint and format check before version release
Release / test (push) Successful in 2m7s
Release / release (push) Successful in 1m49s
Release / pkg-aur (push) Successful in 50s
Release / pkg-deb (push) Successful in 1m30s
Release / pkg-rpm (push) Successful in 1m48s
2026-10-02 23:38:28 +02:00
warkanum b7ae00ab52 fix(test): simplify parameter list in identity column builder 2026-10-02 23:35:56 +02:00
warkanum 8f8664357e chore(release): update package version to 1.0.85
Release / test (push) Failing after 2m1s
Release / release (push) Skipped
Release / pkg-aur (push) Skipped
Release / pkg-deb (push) Skipped
Release / pkg-rpm (push) Skipped
2026-10-02 23:32:07 +02:00
warkanum 7bd61a21b5 fix: resolve gofumpt and staticcheck lint errors 2026-10-02 23:31:41 +02:00
warkanum 6f6b9834ca docs(CONTRIBUTING): update setup instructions and add make targets 2026-10-02 23:29:25 +02:00
warkanum 9cc10715e3 feat: generated and identity columns for mssql, sqlite, drizzle, typeorm, bun and gorm readers
- mssql: write computed columns (AS (expr) PERSISTED) and identity; read computed definition and identity
- sqlite: write GENERATED ALWAYS AS (expr) STORED; read generated columns via table_xinfo and parse the expression
- drizzle: generatedAlwaysAs / generated(Always|ByDefault)AsIdentity, read and write
- typeorm: asExpression/generatedType and identity decorators, read and write; decorator scan is now quote-aware
- bun, gorm readers: read generated and identity markers
- readmes updated
2026-10-02 23:04:04 +02:00
warkanum bbab5ce936 feat: mark generated and identity columns across writers and DBML
- bun: scanonly+generated for GENERATED columns; scanonly+identity for ALWAYS identity (PK gets identity only)
- gorm: <-:false+generated / identity, same primary key rule
- dbml: write GENERATED/IDENTITY column notes and read them back into the model
- pgsql: emit generation/identity clauses in migration create-table and add-column, no DEFAULT
- readmes updated
2026-10-02 22:58:00 +02:00
warkanum 275424c605 feat(pgsql): diff against the live database by default for direct output
Direct pgsql output (job output.conn_env, merge --output-conn) now reads the live
schema and executes only the differences. full_ddl: true (job option) or --full-ddl
(merge) restores the full idempotent DDL; file output is unchanged.

- compare PKs by columns, skip constraint-backed indexes, normalize index method,
  FK actions, serial/numeric types, default literals/casts and truncated names
- diff table and column comments instead of re-emitting them
- remove leftover ZZDUMP debug code
2026-10-02 22:52:59 +02:00
warkanum b6c0cd3d1b fix(pgsql-reader): pair composite foreign key columns by position
key_column_usage and constraint_column_usage were joined on constraint name only,
so an N-column foreign key returned N*N column pairs. Read from pg_constraint with
unnest(conkey, confkey) instead.
2026-10-02 22:52:53 +02:00
warkanum d7d1d99ebc fix(pgsql): tie PK sequence to nextval default, setval past data, keep serial defaults
- derive the primary key sequence from the column's nextval() default instead of an unused identity_<table>_<pk> sequence
- setval after table creation (full and diff paths), forward-only, MAX+1 with is_called=false
- do not DROP DEFAULT on serial/bigserial columns without a model default
2026-10-02 22:37:37 +02:00
warkanum 43849324ce chore(release): update package version to 1.0.84
Release / test (push) Successful in 1m25s
Release / release (push) Successful in 2m7s
Release / pkg-aur (push) Successful in 55s
Release / pkg-deb (push) Successful in 1m42s
Release / pkg-rpm (push) Successful in 1m49s
2026-09-24 15:41:46 +02:00
warkanum e1df3b7cee fix(writers): drop version stamps from generated output
Generated files carried the RelSpec version and build date, so every
rebuild produced a diff even when the schema was unchanged.

- bun, gorm, drizzle: remove the GeneratedBy template field and line
- pgsql, mssql, sqlite: emit a constant "-- Generated by RelSpec"
- bun: replace the version header test with a version-free assertion

pkg/buildinfo is unchanged and still backs the CLI version.
2026-09-24 15:40:37 +02:00
warkanum f7b5d5f054 chore(release): update package version to 1.0.83
Release / pkg-deb (push) Successful in 1m47s
Release / test (push) Successful in 59s
Release / release (push) Successful in 2m0s
Release / pkg-aur (push) Successful in 54s
Release / pkg-rpm (push) Successful in 1m56s
2026-09-23 18:52:48 +02:00
warkanum 99fc4b0944 Merge pull request 'Fix/dbml commented refs duplicate indexes' (#31) from fix/dbml-commented-refs-duplicate-indexes into master
Reviewed-on: #31
2026-09-23 16:48:54 +00:00
warkanum 278d488363 chore(lint): use strings.EqualFold in jobs output format check 2026-09-23 18:48:07 +02:00
warkanum 2f69205aa0 fix(dbml): resolve commented cross-file // Ref: lines
Commented refs are collected per file and resolved against the combined
model after all inputs are loaded (directory, --from-list, merge, jobs).
Matched refs become FKs and relationships; duplicates of existing FKs are
skipped; missing targets are skipped with a warning; column type
mismatches warn.

Also keep reused index names within a DBML table instead of overwriting,
give a second FK to the same table a distinct relationship name, and make
the pgsql writer match relationships to FKs by name first.
2026-09-23 18:48:07 +02:00
warkanum b91985c493 feat(inspector): fail on duplicate index names per schema
PostgreSQL index names share the schema-wide relation namespace, so a
reused name makes CREATE INDEX IF NOT EXISTS silently skip the duplicate.
Add duplicate_index_name rule (enforce by default) covering indexes and
PK/unique constraint backing indexes.
2026-09-23 18:48:07 +02:00
warkanum 80a3453233 Merge pull request 'fix(jobs): preserve writer and template output options' (#30) from issue-29-job-manifest-options into master
Reviewed-on: #30
Reviewed-by: Warky <2+warkanum@noreply@warky.dev>
2026-09-21 04:00:50 +00:00
SG Command ade71b75fa fix(jobs): preserve writer and template output options 2026-09-20 23:16:41 +02:00
warkanum 799b7feb53 ci(rpm): update RPM builder to avoid transient mirrors 2026-09-20 21:41:52 +02:00
warkanum eb40201b59 chore(release): update package version to 1.0.82
Release / release (push) Successful in 1m50s
Release / pkg-aur (push) Successful in 1m0s
Release / pkg-deb (push) Successful in 1m17s
Release / test (push) Successful in 53s
Release / pkg-rpm (push) Failing after 16s
2026-09-20 20:37:41 +02:00
warkanum 1461f8f69f chore(release): update version to 1.0.81 and metadata 2026-09-20 20:37:09 +02:00
warkanum ea70e19a46 feat(jobs): expand environment variables in paths 2026-09-20 20:08:20 +02:00
warkanum 52f4642d97 feat(models): add support for generated and identity columns
Release / test (push) Successful in 3m13s
Release / release (push) Successful in 1m52s
Release / pkg-aur (push) Successful in 1m39s
Release / pkg-deb (push) Successful in 2m15s
Release / pkg-rpm (push) Failing after 46s
* extend Column struct with Generated, GenerationExpression, Identity, and IdentityGeneration fields
* update queryColumns to handle new column properties
* modify migration writer to avoid altering generated and identity columns
* add tests to ensure correct handling of generated and identity columns
2026-09-20 18:33:12 +02:00
warkanum a84592374c Merge pull request 'Feat generated headers column order' (#28) from feat-generated-headers-column-order into master
Release / pkg-rpm (push) Successful in 3m58s
Release / release (push) Successful in 1m54s
Release / test (push) Successful in 5m49s
Release / pkg-aur (push) Successful in 7m13s
Release / pkg-deb (push) Successful in 5m8s
Reviewed-on: #28
2026-09-10 20:22:50 +00:00
warkanum ca226e83df feat(release): update release process to include linting and testing 2026-09-10 22:21:40 +02:00
warkanum 19a2cc1fe3 feat(writers): stamp RelSpec version in generated headers; preserve DBML column order
- Add pkg/buildinfo with Version/BuildDate (set via ldflags, VCS fallback);
  cmd/relspec now sources version info from it
- Emit "RelSpec <version> (built: <date>)" in generated file headers for
  bun, gorm, drizzle, pgsql (incl. migration), mssql, sqlite writers
- pgsql writer: getSortedColumns now sorts by Sequence then Name so the
  streaming WriteSchema/WriteDatabase path preserves source column order
- dbml reader: mergeTable re-bases merged-in column Sequence values past the
  existing max, fixing colliding sequences (and alphabetical fallback) when a
  table is split across multiple DBML files
- Tests for bun/gorm header + column order, and dbml multi-file merge ordering
2026-09-10 22:08:49 +02:00
warkanum 58e46e5b59 feat(dbml): support case-insensitive table note parsing
Release / test (push) Successful in 4m15s
Release / release (push) Successful in 5m29s
Release / pkg-rpm (push) Successful in 2m10s
Release / pkg-deb (push) Successful in 2m27s
Release / pkg-aur (push) Successful in 5m52s
2026-09-09 22:31:00 +02:00
warkanum 161ac317f0 feat(dbml): support multiline table notes in DBML
Release / test (push) Successful in 1m45s
Release / release (push) Successful in 12m24s
Release / pkg-rpm (push) Successful in 2m3s
Release / pkg-deb (push) Successful in 2m17s
Release / pkg-aur (push) Successful in 2m47s
* Add parsing for triple-quoted table notes in DBML
* Update writer to format multiline notes correctly
* Enhance tests for multiline table notes handling
2026-09-09 21:38:11 +02:00
warkanum ee5c009234 feat(main): add silent mode to suppress output messages
* implement --silent flag to control output verbosity
* update error handling to restore stderr after silent mode
* enhance progress reporting in PostgreSQL reader
2026-09-09 21:30:15 +02:00
warkanum 9e9a17d578 Merge pull request 'feat(dbml): @postgres/@sqlite dialect directives (#19)' (#27) from issue-19-dbml-directives into master
Release / test (push) Successful in 58s
Release / release (push) Successful in 5m13s
Release / pkg-aur (push) Successful in 1m0s
Release / pkg-rpm (push) Successful in 2m55s
Release / pkg-deb (push) Successful in 3m21s
Reviewed-on: #27
2026-09-08 14:19:02 +00:00
warkanum 7b628b888c Merge pull request 'feat(job): complete deferred job-file features (#20)' (#26) from issue-20-complete-job-files into master
Reviewed-on: #26
2026-09-08 14:18:57 +00:00
HeinandClaude Sonnet 5 ce3b615b0a feat(dbml): @postgres/@sqlite dialect directives (#19)
Add parseable `@<namespace>[(<target>)]: <args>` directives embedded in DBML.
They are stored losslessly on each object's Metadata, round-trip unchanged
through the DBML writer, and are translated to SQL only by the writer for the
matching dialect.

- models: Directive type + catalog; Metadata map added to Column and Index
- dbml reader: parse and attach directives at database/table/column/index
  level; line-numbered errors; repeatable by default with singleton duplicate
  detection. Fixes a preexisting bug where an `indexes {}` closing brace ended
  the table early, dropping trailing Note: and directive lines.
- dbml writer: re-emit directives at their location; idempotent output
- pgsql writer: PARTITION BY / INHERITS / WITH / TABLESPACE (table),
  STORAGE / COMPRESSION / identity (column), WITH / TABLESPACE (index)
- sqlite writer: WITHOUT ROWID / STRICT (table), COLLATE (column)
- --strict-directives flag on ReaderOptions and WriterOptions
- docs/DBML_DIRECTIVES.md + reader/writer READMEs

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Ss2MY5J11cRGwEz86ZXk7d
2026-09-08 16:17:37 +02:00
Hein 84a6b31873 feat(job): complete deferred job-file features
Implements the remaining items from issue #20:

- version is now forward-permissive: any value >= 1 is accepted; a
  newer-than-known version loads best-effort (unknown fields ignored,
  warning printed) instead of hard-failing on "must be 1"
- from_job input reference: `inputs: [{ from_job: <job> }]` resolves to
  that job's single-file output + format and implies a dependency edge;
  combined depends_on + from_job graph gets topological ordering and
  cycle detection
- logfile size-rotation, on by default (5MB, keep 3), overridable per
  job (log_max_size / log_keep) or file-wide via a top-level defaults block
- new commands: split (schema/table subsetting via select:), inspect
  (rule validation -> markdown/json report, fails job on enforced-rule
  errors), diff (compare exactly two schemas, never fails), scripts-exec
  (run SQL script dirs against a live PostgreSQL database)
- atomic single-file output/report writes (temp file + rename)
- symlink-escape hardening in SafeJoin via EvalSymlinks preflight

Updates docs/JOB_FILES.md and examples/jobs/relspec.yml accordingly.
2026-09-08 14:32:07 +02:00
warkanum f968e3d4a6 Merge pull request 'feat(job): support templ command' (#25) from issue-20-templ-job-command into master
Reviewed-on: #25
2026-09-08 11:04:33 +00:00
SG Command 3d57c947cd feat(job): support templ command 2026-09-08 06:17:45 +02:00
warkanum cdd066dafe fix(sqltypes): serialize SqlTimeStamp as RFC3339
Release / test (push) Successful in 2m22s
Release / pkg-rpm (push) Successful in 1m37s
Release / release (push) Successful in 8m23s
Release / pkg-aur (push) Successful in 48s
Release / pkg-deb (push) Successful in 1m31s
Marshal JSON/YAML/XML and driver Value now emit time.RFC3339 instead of
the offset-less "2006-01-02T15:04:05" layout; zero-time guards compare
against the RFC3339 zero value. Parsing is unchanged and still accepts
bare datetimes.
2026-09-03 22:12:36 +02:00
warkanum 5c25f333b7 fix(go.mod): update Go version to 1.25.13 2026-09-03 21:39:56 +02:00
warkanum 77e6e72f5a fix(ci): don't capture tool-download noise as a formatting diff
The gofumpt check merged stderr into the diff variable, so 'go:
downloading ...' lines made it always fail. Move tool installation into
its own step (adding GOPATH/bin to PATH) and capture only stdout.
2026-09-03 21:33:12 +02:00
warkanum 4d4bc09b86 fix(ci): run lint tools via 'go run' to match the toolchain
Installed staticcheck/govulncheck binaries can fail when built with an
older Go than the module targets, and GOPATH/bin is not always on PATH.
Compile them on demand with 'go run <tool>@latest' in both the workflow
and the Makefile.
2026-09-03 21:25:27 +02:00
warkanum 96281c9f03 chore(ci): add govulncheck, staticcheck, go vet, gofumpt gates
Release / release (push) Skipped
Release / pkg-aur (push) Skipped
Release / pkg-deb (push) Skipped
Release / pkg-rpm (push) Skipped
Release / test (push) Failing after 2m20s
* Add lint/format checks to the Gitea release workflow and Makefile
  (targets: vet, fmt, fmt-check, staticcheck, govulncheck, check)
* Switch .golangci.json formatter from gofmt to gofumpt (extra.group-params)
* Bump golang.org/x/text 0.37.0 -> 0.39.0 for GO-2026-5970; re-vendor
* Fix staticcheck S1011 in pkg/diff; drop unused pgsql writer helpers
* Fix gocritic unnamedResult (pkg/diff) and rangeValCopy (pkg/pgsql)
* Apply gofumpt + goimports formatting across the tree
2026-09-03 21:17:03 +02:00
warkanum d6d0200938 Merge pull request 'feat(job): declarative YAML job files for named relspec workflows (#20)' (#24) from issue-20-job-files into master
Reviewed-on: #24
2026-09-02 04:16:39 +00:00
267 changed files with 12169 additions and 1585 deletions
+37 -6
View File
@@ -20,12 +20,34 @@ jobs:
with:
go-version-file: go.mod
- name: go vet
run: go vet ./...
- name: Install lint tools
run: |
go install github.com/golangci/golangci-lint/v2/cmd/golangci-lint@latest
go install honnef.co/go/tools/cmd/staticcheck@latest
go install golang.org/x/vuln/cmd/govulncheck@latest
echo "$(go env GOPATH)/bin" >> "$GITHUB_PATH"
- name: gofumpt (golangci-lint fmt)
run: |
diff=$(golangci-lint fmt --diff)
if [ -n "$diff" ]; then
echo "$diff"
echo "Formatting issues found. Run: make fmt"
exit 1
fi
- name: staticcheck
run: staticcheck ./...
- name: govulncheck
run: govulncheck ./...
- name: Test
run: go test ./...
- name: Lint
run: go vet ./...
release:
needs: test
runs-on: ubuntu-latest
@@ -41,6 +63,8 @@ jobs:
- name: Build release binaries
run: |
VERSION="${{ github.event.inputs.tag || github.ref_name }}"
BUILD_DATE="$(date -u +"%Y-%m-%d %H:%M:%S UTC")"
LDFLAGS="-X 'git.warky.dev/wdevs/relspecgo/pkg/buildinfo.Version=${VERSION}' -X 'git.warky.dev/wdevs/relspecgo/pkg/buildinfo.BuildDate=${BUILD_DATE}'"
for target in "linux/amd64" "linux/arm64" "darwin/amd64" "darwin/arm64" "windows/amd64"; do
GOOS="${target%/*}"
GOARCH="${target#*/}"
@@ -49,7 +73,7 @@ jobs:
NAME="relspec-${GOOS}-${GOARCH}${EXT}"
GOOS="$GOOS" GOARCH="$GOARCH" go build \
-trimpath \
-ldflags "-X git.warky.dev/wdevs/relspecgo/cmd/relspec.version=${VERSION}" \
-ldflags "$LDFLAGS" \
-o "$NAME" ./cmd/relspec
echo "Built $NAME"
done
@@ -212,11 +236,12 @@ jobs:
run: |
VERSION="${{ github.event.inputs.tag || github.ref_name }}"
PKGVER="${VERSION#v}"
BUILD_DATE="$(date -u +"%Y-%m-%d %H:%M:%S UTC")"
for GOARCH in amd64 arm64; do
GOOS=linux GOARCH=$GOARCH go build \
-trimpath \
-ldflags "-X git.warky.dev/wdevs/relspecgo/cmd/relspec.version=${PKGVER}" \
-ldflags "-X 'git.warky.dev/wdevs/relspecgo/pkg/buildinfo.Version=v${PKGVER}' -X 'git.warky.dev/wdevs/relspecgo/pkg/buildinfo.BuildDate=${BUILD_DATE}'" \
-o relspec ./cmd/relspec
PKGDIR="relspec_${PKGVER}_${GOARCH}"
@@ -287,7 +312,13 @@ jobs:
rockylinux:9 \
bash -lc "
set -euo pipefail
dnf install -y rpm-build git &&
# Avoid transient/out-of-sync mirrors while bootstrapping the RPM builder.
sed -i \
-e 's|^mirrorlist=|#mirrorlist=|' \
-e 's|^#baseurl=http://dl.rockylinux.org|baseurl=https://dl.rockylinux.org|' \
/etc/yum.repos.d/rocky*.repo
dnf clean all
dnf -y --setopt=timeout=60 --setopt=retries=5 install rpm-build git
curl -fsSL https://go.dev/dl/go\${GO_VER}.linux-amd64.tar.gz | tar -C /usr/local -xz &&
export PATH=\$PATH:/usr/local/go/bin &&
mkdir -p ~/rpmbuild/{BUILD,BUILDROOT,RPMS,SOURCES,SPECS,SRPMS} &&
+5 -3
View File
@@ -1,7 +1,7 @@
{
"formatters": {
"enable": [
"gofmt",
"gofumpt",
"goimports"
],
"exclusions": {
@@ -13,8 +13,10 @@
]
},
"settings": {
"gofmt": {
"simplify": true
"gofumpt": {
"extra": {
"group-params": true
}
},
"goimports": {
"local-prefixes": [
+59 -113
View File
@@ -1,145 +1,91 @@
# Contributing to RelSpec
Thank you for your interest in contributing to RelSpec.
## Setup
## Development Setup
- Go 1.25+ (see `go.mod`), Git
- Optional: golangci-lint, Docker/Podman (PostgreSQL integration tests)
### Prerequisites
- Go 1.21 or higher
- Git
- (Optional) golangci-lint for linting
- (Optional) Docker for database testing
### Getting Started
1. Clone the repository:
```bash
git clone https://github.com/wdevs/relspecgo.git
git clone git@git.warky.dev:wdevs/relspecgo.git
cd relspecgo
make deps
make build # outputs build/relspec
```
2. Install dependencies:
```bash
go mod download
```
## Make Targets
3. Run tests:
```bash
go test ./...
```
| Target | Purpose |
|-------------------------|------------------------------------------------------|
| `make test` | Unit tests (race detection, coverage) |
| `make test-integration` | Integration tests (needs `RELSPEC_TEST_PG_CONN`) |
| `make docker-test` | PostgreSQL integration tests via Docker/Podman |
| `make lint` | golangci-lint |
| `make fmt` / `fmt-check`| gofumpt + goimports |
| `make check` | vet, fmt-check, staticcheck, govulncheck |
| `make coverage` | Coverage report |
4. Build the project:
```bash
go build -o relspec ./cmd/relspec
```
Single test: `go test -run TestName ./pkg/readers/dbml`
## Project Structure
## Layout
```
relspecgo/
├── cmd/ # CLI application entry point
├── pkg/
│ ├── readers/ # Input format readers (XML, JSON, DCTX, DB, GORM, Bun)
│ ├── writers/ # Output format writers (GORM, Bun, JSON, YAML)
│ ├── models/ # Internal data models for relations
│ └── transform/ # Transformation and validation logic
├── examples/ # Usage examples and sample files
├── tests/ # Integration tests
└── .claude/ # Claude Code configuration and commands
cmd/relspec/ CLI commands (convert, diff, merge, split, edit, inspect, job, ...)
pkg/models/ Core model: Database > Schema > Table > Column/Constraint/Index/Relationship
pkg/readers/<fmt> One reader per format
pkg/writers/<fmt> One writer per format
pkg/diff, merge, inspector, jobs, transform, ui, pgsql, sqltypes ...
examples/ Sample files
tests/ Integration tests and assets
docs/ Feature docs
```
## Adding New Readers
## Adding a Reader
To add a new input format reader:
1. Create `pkg/readers/<format>/reader.go` with `NewReader(options *readers.ReaderOptions)`.
2. Implement `readers.Reader`: `ReadDatabase`, `ReadSchema`, `ReadTable`.
3. Add `reader_test.go` in the same package.
4. Register the format in the CLI switches (`cmd/relspec/convert.go`, `diff.go`, `edit.go`, ...).
5. Add a `README.md` in the reader directory.
1. Create a new file in `pkg/readers/` (e.g., `myformat_reader.go`)
2. Implement the `Reader` interface:
```go
type Reader interface {
Read(source string) (*models.Schema, error)
}
```
3. Add tests in `pkg/readers/myformat_reader_test.go`
4. Register the reader in the CLI
## Adding a Writer
## Adding New Writers
1. Create `pkg/writers/<format>/writer.go` with `NewWriter(options *writers.WriterOptions)`.
2. Implement `writers.Writer`: `WriteDatabase`, `WriteSchema`, `WriteTable`.
3. Add `writer_test.go` in the same package.
4. Register the format in the CLI switches.
5. Add a `README.md` in the writer directory.
To add a new output format writer:
## Code Rules
1. Create a new file in `pkg/writers/` (e.g., `myformat_writer.go`)
2. Implement the `Writer` interface:
```go
type Writer interface {
Write(schema *models.Schema, destination string) error
}
```
3. Add tests in `pkg/writers/myformat_writer_test.go`
4. Register the writer in the CLI
## Code Style
- Follow standard Go conventions
- Use `gofmt` for formatting
- Run `go vet` to check for issues
- Use meaningful variable and function names
- Add comments for exported functions and types
- Format with gofumpt/goimports (`make fmt`); `make check` must pass.
- Iterate `Table.Columns`, `Constraints`, `Indexes`, `Relationships` in sorted order (maps are unordered; output must be deterministic).
- Every writer stamps `buildinfo.GeneratedComment()` in its file header.
- Comment exported functions and types.
## Testing
- Write unit tests for all new functionality
- Aim for >80% code coverage
- Use table-driven tests where appropriate
- Include both positive and negative test cases
- Tests live in the same package as the code.
- Table-driven tests; cover positive and negative cases.
- Reuse existing test data in `tests/` and `examples/` before adding new data.
- Tests run with `-race`.
### Running Tests
## Commits
```bash
# All tests
go test ./...
- Format: `type(scope): description`
- Types: `feat`, `fix`, `docs`, `test`, `refactor`, `chore`, `ci`
- Keep commits focused. Reference issues where applicable.
# With coverage
go test -cover ./...
## Pull Requests
# Verbose output
go test -v ./...
1. Branch from `master`.
2. Add tests; `make test` and `make check` pass.
3. Update docs/README if behaviour changes.
4. Open a PR with a clear description.
# Specific package
go test ./pkg/readers/...
```
## Security
## Committing Changes
- Write clear, descriptive commit messages
- Follow conventional commits format: `type(scope): description`
- Types: feat, fix, docs, test, refactor, chore
- Example: `feat(readers): add PostgreSQL support`
- Keep commits focused and atomic
- Reference issues in commit messages when applicable
## Pull Request Process
1. Create a feature branch from `master`
2. Make your changes
3. Add tests for new functionality
4. Ensure all tests pass
5. Update documentation if needed
6. Submit a pull request with a clear description
## Claude Code Commands
This project includes Claude Code slash commands for common tasks:
- `/test` - Run all tests
- `/build` - Build the binary
- `/lint` - Run linters
- `/coverage` - Generate coverage report
## Questions or Issues?
- Open an issue for bugs or feature requests
- Start a discussion for questions or ideas
- Check existing issues before creating new ones
Do not report vulnerabilities in public issues. See [SECURITY.md](SECURITY.md).
## License
By contributing to RelSpec, you agree that your contributions will be licensed under the Apache License 2.0.
Contributions are licensed under the Apache License 2.0.
+60 -12
View File
@@ -1,4 +1,4 @@
.PHONY: all build test test-unit test-integration lint coverage clean install help docker-up docker-down docker-test docker-test-integration start stop release release-version godoc
.PHONY: all build test test-unit test-integration lint coverage clean install help docker-up docker-down docker-test docker-test-integration start stop release release-version rerelease godoc vet fmt fmt-check staticcheck govulncheck check
# Binary name
BINARY_NAME=relspec
@@ -14,10 +14,17 @@ GOGET=$(GOCMD) get
GOMOD=$(GOCMD) mod
GOCLEAN=$(GOCMD) clean
# Tool versions (compiled on demand via `go run` so they match the local toolchain)
GOLANGCI_LINT = go run github.com/golangci/golangci-lint/v2/cmd/golangci-lint@latest
STATICCHECK = go run honnef.co/go/tools/cmd/staticcheck@latest
GOVULNCHECK = go run golang.org/x/vuln/cmd/govulncheck@latest
# Version information
VERSION := $(shell git describe --tags --always --dirty 2>/dev/null || echo "dev")
# VERSION file is the single source of truth (bumped by `make release-version`).
# Falls back to git describe for ad-hoc builds where VERSION hasn't been committed yet.
VERSION := $(shell [ -f VERSION ] && echo "v$$(cat VERSION | tr -d '[:space:]')" || git describe --tags --always --dirty 2>/dev/null || echo "dev")
BUILD_DATE := $(shell date -u +"%Y-%m-%d %H:%M:%S UTC")
LDFLAGS := -X 'main.version=$(VERSION)' -X 'main.buildDate=$(BUILD_DATE)'
LDFLAGS := -X 'git.warky.dev/wdevs/relspecgo/pkg/buildinfo.Version=$(VERSION)' -X 'git.warky.dev/wdevs/relspecgo/pkg/buildinfo.BuildDate=$(BUILD_DATE)'
# Auto-detect container runtime (Docker or Podman)
CONTAINER_RUNTIME := $(shell \
@@ -41,6 +48,29 @@ COMPOSE_CMD := $(shell \
all: lint test build ## Run linting, tests, and build
check: vet fmt-check staticcheck govulncheck ## Run vet, gofumpt check, staticcheck, and govulncheck
vet: ## Run go vet
@echo "Running go vet..."
$(GOCMD) vet ./...
fmt: ## Format code (gofumpt + goimports via golangci-lint)
@echo "Formatting..."
$(GOLANGCI_LINT) fmt --config=.golangci.json
fmt-check: ## Check formatting (gofumpt + goimports via golangci-lint)
@echo "Checking formatting..."
@diff=$$($(GOLANGCI_LINT) fmt --diff --config=.golangci.json); \
if [ -n "$$diff" ]; then echo "$$diff"; echo "Run: make fmt"; exit 1; fi
staticcheck: ## Run staticcheck
@echo "Running staticcheck..."
$(STATICCHECK) ./...
govulncheck: ## Run govulncheck
@echo "Running govulncheck..."
$(GOVULNCHECK) ./...
build: deps ## Build the binary
@echo "Building $(BINARY_NAME) $(VERSION)..."
@mkdir -p $(BUILD_DIR)
@@ -179,7 +209,7 @@ docker-test-integration: docker-up ## Start DB and run integration tests
$(GOTEST) -v ./pkg/readers/pgsql/ -count=1 || (make docker-down && exit 1)
@make docker-down
release: ## Create and push a new release tag (auto-increments patch version)
release: lint fmt-check test build ## Run lint, format check, tests, build, then create and push a new release tag
@echo "Creating new release..."
@latest_tag=$$(git describe --tags --abbrev=0 2>/dev/null || echo ""); \
if [ -z "$$latest_tag" ]; then \
@@ -195,6 +225,15 @@ release: ## Create and push a new release tag (auto-increments patch version)
echo "Creating new release: $$version"; \
commit_logs=$$(git log "$${latest_tag}..HEAD" --pretty=format:"- %s" --no-merges); \
fi; \
PKGVER=$${version#v}; \
echo "Updating package metadata to $$PKGVER..."; \
echo "$$PKGVER" > VERSION; \
sed -i "s/^pkgver=.*/pkgver=$$PKGVER/" linux/arch/PKGBUILD; \
sed -i "s/^Version:.*/Version: $$PKGVER/" linux/centos/relspec.spec; \
git add VERSION linux/arch/PKGBUILD linux/centos/relspec.spec; \
if ! git diff --cached --quiet; then \
git commit -m "chore(release): update package version to $$PKGVER"; \
fi; \
if [ -z "$$commit_logs" ]; then \
tag_message="Release $$version"; \
else \
@@ -204,21 +243,30 @@ release: ## Create and push a new release tag (auto-increments patch version)
git push origin "$$version"; \
echo "Tag $$version created and pushed to remote repository."
release-version: ## Auto-increment patch version, update package files, commit, tag, and push
@CURRENT=$$(git describe --tags --abbrev=0 2>/dev/null || echo "v0.0.0"); \
MAJOR=$$(echo $$CURRENT | sed 's/v\([0-9]*\)\.\([0-9]*\)\.\([0-9]*\).*/\1/'); \
MINOR=$$(echo $$CURRENT | sed 's/v\([0-9]*\)\.\([0-9]*\)\.\([0-9]*\).*/\2/'); \
PATCH=$$(echo $$CURRENT | sed 's/v\([0-9]*\)\.\([0-9]*\)\.\([0-9]*\).*/\3/'); \
NEXT="v$$MAJOR.$$MINOR.$$((PATCH + 1))"; \
release-version: lint fmt-check ## Run lint and format check, then auto-increment patch version, update VERSION/package files, commit, tag, and push
@CURRENT=$$([ -f VERSION ] && cat VERSION | tr -d '[:space:]' || (git describe --tags --abbrev=0 2>/dev/null | sed 's/^v//') || echo "0.0.0"); \
MAJOR=$$(echo $$CURRENT | cut -d. -f1); \
MINOR=$$(echo $$CURRENT | cut -d. -f2); \
PATCH=$$(echo $$CURRENT | cut -d. -f3); \
PKGVER="$$MAJOR.$$MINOR.$$((PATCH + 1))"; \
echo "Current: $$CURRENT → Next: $$NEXT"; \
NEXT="v$$PKGVER"; \
echo "Current: v$$CURRENT → Next: $$NEXT"; \
echo "$$PKGVER" > VERSION; \
sed -i "s/^pkgver=.*/pkgver=$$PKGVER/" linux/arch/PKGBUILD; \
sed -i "s/^Version:.*/Version: $$PKGVER/" linux/centos/relspec.spec; \
git add linux/arch/PKGBUILD linux/centos/relspec.spec; \
git add VERSION linux/arch/PKGBUILD linux/centos/relspec.spec; \
git commit -m "chore(release): update package version to $$PKGVER"; \
git tag -a "$$NEXT" -m "Release $$NEXT"; \
git push origin HEAD "$$NEXT"; \
echo "Pushed $$NEXT — release workflow triggered"
rerelease: lint fmt-check ## Move the latest tag to HEAD and force push it
@TAG=$$(git describe --tags --abbrev=0 2>/dev/null); \
if [ -z "$$TAG" ]; then echo "No existing tags found"; exit 1; fi; \
echo "Moving $$TAG to $$(git rev-parse --short HEAD)"; \
git tag -f -a "$$TAG" -m "Release $$TAG" HEAD; \
git push --force origin "$$TAG"; \
echo "Pushed $$TAG — release workflow triggered"
help: ## Display this help screen
@grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-20s\033[0m %s\n", $$1, $$2}'
+15 -2
View File
@@ -23,6 +23,8 @@ go install -v git.warky.dev/wdevs/relspecgo/cmd/relspec@latest
| **Readers** | `bun` `dbml` `dctx` `drawdb` `drizzle` `gorm` `graphql` `json` `mssql` `pgsql` `prisma` `sqldir` `sqlite` `typeorm` `yaml` |
| **Writers** | `bun` `dbml` `dctx` `drawdb` `drizzle` `gorm` `graphql` `json` `mssql` `pgsql` `prisma` `sqlexec` `sqlite` `template` `typeorm` `yaml` |
See [docs/FORMAT_EXAMPLES.md](docs/FORMAT_EXAMPLES.md) for usage examples covering every format.
## Commands
### `convert` — Schema conversion
@@ -40,8 +42,17 @@ relspec convert --from pgsql --from-conn "postgres://..." --to sqlite --to-path
# Multiple input files merged
relspec convert --from json --from-list "a.json,b.json" --to yaml --to-path merged.yaml
# Watch mode: regenerate whenever the source file(s) change (Ctrl-C to stop)
relspec convert --from dbml --from-path schema.dbml --to gorm --to-path models/ --package models --watch
```
`--watch` works with `--from-path` and `--from-list` (not live database
connections or `--dry-run`). Source files are polled every `--watch-interval`
(default 500ms), a directory source is watched recursively, and the output path
is ignored so generating into the source tree does not loop. Conversion errors
are printed and watching continues.
PostgreSQL connections opened by relspec set `application_name` by default to
`relspecgo/<version>` (with component suffixes internally, e.g. readers/writers).
If you need a custom value, provide `application_name` explicitly in the connection
@@ -126,7 +137,7 @@ relspec job run build-schema
version: 1
jobs:
build-schema:
command: convert # closed allow-list: convert | merge | scripts-list
command: convert # closed allow-list: convert | merge | split | scripts-list | scripts-exec | templ | inspect | diff
description: Merge the DBML sources and emit PostgreSQL DDL
inputs:
- path: schema/core.dbml
@@ -147,7 +158,9 @@ resolved relative to the job file and may not escape it, and remote database
credentials are referenced by environment-variable name (`conn_env:`) and
redacted from logs. The whole plan — unknown commands/formats, duplicate job
names, missing inputs, path traversal, dependency cycles — is validated before
any job runs. See [docs/JOB_FILES.md](docs/JOB_FILES.md).
any job runs. Path-like fields also support `${NAME}` environment-variable
references, which are expanded and safety-checked during pre-flight. See
[docs/JOB_FILES.md](docs/JOB_FILES.md).
### `edit` — Interactive TUI editor
+12
View File
@@ -0,0 +1,12 @@
# Security Policy
## Supported versions
Security fixes are released for the latest minor version of RelSpec.
## Reporting a vulnerability
Please do not open a public issue for security problems.
Report privately by email: warkydevs@gmail.com
Reports are reviewed on a best-effort basis. No response time or fix timeline is
guaranteed. Reporters may be credited in the release notes if they wish.
+20 -9
View File
@@ -4,20 +4,25 @@
- [✔️] **Database Inspector**
- [✔️] PostgreSQL driver (reader + writer)
- [ ] MySQL driver
- [ ] MySQL driver (only MariaDB datatype conversion in pkg/mariadb, no reader/writer)
- [✔️] SQLite driver (reader + writer with automatic schema flattening)
- [ ] MSSQL driver
- [✔️] MSSQL driver (reader + writer, generated and identity columns)
- [✔️] Foreign key detection
- [✔️] Index extraction
- [✔️] .sql file generation (PostgreSQL, SQLite)
- [✔️] .dbml: Database Markup Language (DBML) for textual schema representation.
- [✔️] Prisma schema support (PSL format) .prisma
- [ ] Sequelize (Typescript/Javascript) (Use templates, 💲 Someone can do this, not me)
- [✔️] Drizzle ORM support .ts (TypeScript / JavaScript) (Mr. Edd wanted to move from Prisma to Drizzle. If you are bugs, you are welcome to do pull requests or issues)
- [☠️] Entity Framework (.NET) model .edmx (Fuck no, EDMX files were bloated, verbose XML nightmares—hard to merge, error-prone, and a pain in teams. Microsoft wisely ditched them in EF Core for code-first. Classic overkill from old MS era.)
- [✔️] TypeORM support
- [] .hbm.xml / schema.xml: Hibernate/Propel mappings (Java/PHP) (💲 Someone can do this, not me)
- [ ] Django models.py (Python classes), Sequelize migrations (JS) (💲 Someone can do this, not me)
- [] .avsc: Avro schema (JSON format for data serialization) (💲 Someone can do this, not me)
- [ ] .hbm.xml / schema.xml: Hibernate/Propel mappings (Java/PHP) (Use templates, 💲 Someone can do this, not me)
- [ ] Django models.py (Python classes), Sequelize migrations (JS) (Use templates, 💲 Someone can do this, not me)
- [ ] SQLAlchemy, Python (Use templates, 💲 Someone can do this, not me)
- [ ] .avsc: Avro schema (JSON format for data serialization) (Use templates, 💲 Someone can do this, not me)
- [ ] Rails schema in db/schema.rb (Use templates, 💲 Someone can do this, not me)
- [ ] Laravel migration (Use templates, 💲 Someone can do this, not me)
- [ ] Hibernate / JPA (Use templates, 💲 Someone can do this, not me)
- [✔️] GraphQL schema generation
## UI
@@ -39,13 +44,19 @@
## Advanced Features
- [ ] Dry-run mode for validation
- [x] Diff tool for comparing specifications
- [ ] Migration script generation
- [ ] Dry-run mode for validation (only `job run --dry-run`; not on convert/merge/split)
- [✔️] Diff tool for comparing specifications
- [✔️] Migration script generation (PostgreSQL diff, live database by default for direct output)
- [ ] Custom type mapping configuration
- [ ] Batch processing support
- [ ] Batch processing support (partial: job files run named workflows via `relspec job run`)
- [ ] Watch mode for auto-regeneration
## Distribution
- [ ] NSIS Windows installer
- [ ] Check https://git.warky.dev/wdevs/relspecgo/releases for new releases and prompt to update
- [ ] CI pipeline to build and publish the Windows installer
## Future Considerations
- [ ] Web UI for visual editing
+1
View File
@@ -0,0 +1 @@
1.0.85
+142 -15
View File
@@ -3,6 +3,7 @@ package main
import (
stdjson "encoding/json"
"fmt"
"io"
"os"
"strings"
"time"
@@ -57,6 +58,9 @@ var (
convertNullableArrays string
convertContinueOnError bool
convertExtraFields string
convertDryRun bool
convertWatch bool
convertWatchInterval time.Duration
)
var convertCmd = &cobra.Command{
@@ -165,7 +169,11 @@ Examples:
# Convert SQLite to PostgreSQL SQL
relspec convert --from sqlite --from-path database.db \
--to pgsql --to-path schema.sql`,
--to pgsql --to-path schema.sql
# Regenerate GORM models every time the DBML file changes
relspec convert --from dbml --from-path schema.dbml \
--to gorm --to-path models/ --package models --watch`,
RunE: runConvert,
}
@@ -185,6 +193,11 @@ func init() {
convertCmd.Flags().BoolVar(&convertContinueOnError, "continue-on-error", false, "Prepend \\set ON_ERROR_STOP off to generated SQL so psql continues past errors (pgsql output only)")
convertCmd.Flags().StringVar(&convertExtraFields, "extra-fields", "", "Path to JSON file containing extra Bun model fields to inject (bun output only); fields support target_table, name, type, bun_tag, json_tag, comment")
convertCmd.Flags().BoolVar(&convertDryRun, "dry-run", false, "Read and validate the input and print the plan without writing any output")
convertCmd.Flags().BoolVar(&convertWatch, "watch", false, "Watch the source files (--from-path or --from-list) and regenerate the output whenever they change")
convertCmd.Flags().DurationVar(&convertWatchInterval, "watch-interval", 500*time.Millisecond, "Polling interval used by --watch")
err := convertCmd.MarkFlagRequired("from")
if err != nil {
fmt.Fprintf(os.Stderr, "Error marking from flag as required: %v\n", err)
@@ -200,6 +213,13 @@ func init() {
}
func runConvert(cmd *cobra.Command, args []string) error {
if convertWatch {
return runConvertWatch(cmd.Context(), os.Stderr, func() error { return runConvertOnce(cmd) })
}
return runConvertOnce(cmd)
}
func runConvertOnce(cmd *cobra.Command) error {
fmt.Fprintf(os.Stderr, "\n=== RelSpec Schema Converter ===\n")
fmt.Fprintf(os.Stderr, "Started at: %s\n\n", getCurrentTimestamp())
@@ -229,6 +249,7 @@ func runConvert(cmd *cobra.Command, args []string) error {
if err != nil {
return fmt.Errorf("failed to read source: %w", err)
}
finalizeCommentedRefs(db, stderrWarn)
fmt.Fprintf(os.Stderr, " ✓ Successfully read database '%s'\n", db.Name)
fmt.Fprintf(os.Stderr, " Found: %d schema(s)\n", len(db.Schemas))
@@ -239,6 +260,22 @@ func runConvert(cmd *cobra.Command, args []string) error {
}
fmt.Fprintf(os.Stderr, " Found: %d table(s)\n\n", totalTables)
if convertDryRun {
if err := validateWriteTarget(db, convertTargetType, convertPackageName, convertSchemaFilter, convertExtraFields); err != nil {
return fmt.Errorf("dry run validation failed: %w", err)
}
out := outWriter(cmd)
fmt.Fprintf(out, "RelSpec convert plan (dry run - nothing written):\n")
fmt.Fprintf(out, " Input: %s database '%s'\n", convertSourceType, db.Name)
fmt.Fprintf(out, " Output: %s -> %s\n", convertTargetType, convertTargetPath)
if convertSchemaFilter != "" {
fmt.Fprintf(out, " Schema filter: %s\n", convertSchemaFilter)
}
printDryRunPlan(out, db)
fmt.Fprintf(os.Stderr, "=== Dry run complete: no output written ===\n\n")
return nil
}
// Write to target format
fmt.Fprintf(os.Stderr, "[2/2] Writing to target format...\n")
fmt.Fprintf(os.Stderr, " Format: %s\n", convertTargetType)
@@ -261,6 +298,18 @@ func runConvert(cmd *cobra.Command, args []string) error {
return nil
}
// finalizeCommentedRefs resolves DBML `// Ref:` comments against the fully
// loaded model and warns about refs whose target is not loaded.
func finalizeCommentedRefs(db *models.Database, warn func(string)) {
for _, w := range dbml.ResolveCommentedRefs(db, true) {
warn(w)
}
}
func stderrWarn(msg string) {
fmt.Fprintf(os.Stderr, " ⚠ %s\n", msg)
}
func readDatabaseListForConvert(dbType string, files []string) (*models.Database, error) {
if len(files) == 0 {
return nil, fmt.Errorf("file list is empty")
@@ -395,23 +444,12 @@ func writeDatabase(db *models.Database, dbType, outputPath, packageName, schemaF
writerOpts := newWriterOptions(outputPath, packageName, flattenSchema, nullableTypes, nullableArrays, continueOnError)
if extraFields != "" {
if !strings.EqualFold(dbType, "bun") {
return fmt.Errorf("--extra-fields is only supported for Bun output")
}
extraFieldsJSON, err := os.ReadFile(extraFields)
extraFieldsJSON, err := loadExtraFields(dbType, extraFields)
if err != nil {
return fmt.Errorf("failed to read --extra-fields file %q: %w", extraFields, err)
}
var parsed []wbun.ExtraFieldConfig
if err := stdjson.Unmarshal(extraFieldsJSON, &parsed); err != nil {
return fmt.Errorf("invalid --extra-fields JSON in %q: %w", extraFields, err)
}
if len(parsed) == 0 {
return fmt.Errorf("--extra-fields must contain at least one field")
return err
}
writerOpts.Metadata = map[string]interface{}{
"extra_fields": string(extraFieldsJSON),
"extra_fields": extraFieldsJSON,
}
}
@@ -516,6 +554,95 @@ func writeDatabase(db *models.Database, dbType, outputPath, packageName, schemaF
return nil
}
// loadExtraFields reads and validates the --extra-fields JSON file, returning
// its raw content.
func loadExtraFields(dbType, path string) (string, error) {
if !strings.EqualFold(dbType, "bun") {
return "", fmt.Errorf("--extra-fields is only supported for Bun output")
}
extraFieldsJSON, err := os.ReadFile(path)
if err != nil {
return "", fmt.Errorf("failed to read --extra-fields file %q: %w", path, err)
}
var parsed []wbun.ExtraFieldConfig
if err := stdjson.Unmarshal(extraFieldsJSON, &parsed); err != nil {
return "", fmt.Errorf("invalid --extra-fields JSON in %q: %w", path, err)
}
if len(parsed) == 0 {
return "", fmt.Errorf("--extra-fields must contain at least one field")
}
return string(extraFieldsJSON), nil
}
// validateWriteTarget performs the checks writeDatabase makes before writing,
// without constructing a writer or touching the output path. Used by --dry-run.
func validateWriteTarget(db *models.Database, dbType, packageName, schemaFilter, extraFields string) error {
if extraFields != "" {
if _, err := loadExtraFields(dbType, extraFields); err != nil {
return err
}
}
switch strings.ToLower(dbType) {
case "dbml", "dctx", "drawdb", "json", "yaml", "yml", "drizzle",
"pgsql", "postgres", "postgresql", "sql",
"mssql", "sqlserver", "mssql2016", "mssql2017", "mssql2019", "mssql2022",
"sqlite", "sqlite3", "prisma", "typeorm", "graphql", "gql":
case "gorm", "bun":
if packageName == "" {
return fmt.Errorf("package name is required for %s format (use --package flag)", dbType)
}
default:
return fmt.Errorf("unsupported target format: %s", dbType)
}
if schemaFilter != "" {
for _, schema := range db.Schemas {
if schema.Name == schemaFilter {
return nil
}
}
return fmt.Errorf("schema '%s' not found in database. Available schemas: %v",
schemaFilter, getSchemaNames(db))
}
if strings.EqualFold(dbType, "dctx") {
if len(db.Schemas) == 0 {
return fmt.Errorf("no schemas found in database")
}
if len(db.Schemas) > 1 {
return fmt.Errorf("multiple schemas found, please specify which schema to export using --schema flag. Available schemas: %v",
getSchemaNames(db))
}
}
return nil
}
// outWriter returns the command's stdout, falling back to os.Stdout when the
// command is nil (tests call the run functions directly).
func outWriter(cmd *cobra.Command) io.Writer {
if cmd == nil {
return os.Stdout
}
return cmd.OutOrStdout()
}
// printDryRunPlan prints the schemas and tables that would be written.
func printDryRunPlan(out io.Writer, db *models.Database) {
for _, schema := range db.Schemas {
names := make([]string, 0, len(schema.Tables))
for _, t := range schema.Tables {
names = append(names, t.Name)
}
fmt.Fprintf(out, " schema %q: %d table(s)", schema.Name, len(schema.Tables))
if len(names) > 0 {
fmt.Fprintf(out, " [%s]", strings.Join(names, ", "))
}
fmt.Fprintln(out)
}
}
// getSchemaNames returns a slice of schema names from a database
func getSchemaNames(db *models.Database) []string {
names := make([]string, len(db.Schemas))
+3 -3
View File
@@ -46,7 +46,7 @@ func TestReadDatabaseListForConvert_MultipleFiles(t *testing.T) {
func TestReadDatabaseListForConvert_PathWithSpaces(t *testing.T) {
spacedDir := filepath.Join(t.TempDir(), "my schema files")
if err := os.MkdirAll(spacedDir, 0755); err != nil {
if err := os.MkdirAll(spacedDir, 0o755); err != nil {
t.Fatal(err)
}
file := filepath.Join(spacedDir, "my users schema.json")
@@ -63,7 +63,7 @@ func TestReadDatabaseListForConvert_PathWithSpaces(t *testing.T) {
func TestReadDatabaseListForConvert_MultipleFilesPathWithSpaces(t *testing.T) {
spacedDir := filepath.Join(t.TempDir(), "my schema files")
if err := os.MkdirAll(spacedDir, 0755); err != nil {
if err := os.MkdirAll(spacedDir, 0o755); err != nil {
t.Fatal(err)
}
file1 := filepath.Join(spacedDir, "users schema.json")
@@ -154,7 +154,7 @@ func TestRunConvert_FromListEndToEndPathWithSpaces(t *testing.T) {
defer restoreConvertState(saved)
spacedDir := filepath.Join(t.TempDir(), "my schema dir")
if err := os.MkdirAll(spacedDir, 0755); err != nil {
if err := os.MkdirAll(spacedDir, 0o755); err != nil {
t.Fatal(err)
}
file1 := filepath.Join(spacedDir, "users schema.json")
+8 -8
View File
@@ -229,43 +229,43 @@ func readDatabase(dbType, filePath, connString, label string) (*models.Database,
if filePath == "" {
return nil, fmt.Errorf("%s: file path is required for DBML format", label)
}
reader = dbml.NewReader(&readers.ReaderOptions{FilePath: filePath})
reader = dbml.NewReader(newReaderOptions(filePath, ""))
case "dctx":
if filePath == "" {
return nil, fmt.Errorf("%s: file path is required for DCTX format", label)
}
reader = dctx.NewReader(&readers.ReaderOptions{FilePath: filePath})
reader = dctx.NewReader(newReaderOptions(filePath, ""))
case "drawdb":
if filePath == "" {
return nil, fmt.Errorf("%s: file path is required for DrawDB format", label)
}
reader = drawdb.NewReader(&readers.ReaderOptions{FilePath: filePath})
reader = drawdb.NewReader(newReaderOptions(filePath, ""))
case "json":
if filePath == "" {
return nil, fmt.Errorf("%s: file path is required for JSON format", label)
}
reader = json.NewReader(&readers.ReaderOptions{FilePath: filePath})
reader = json.NewReader(newReaderOptions(filePath, ""))
case "yaml":
if filePath == "" {
return nil, fmt.Errorf("%s: file path is required for YAML format", label)
}
reader = yaml.NewReader(&readers.ReaderOptions{FilePath: filePath})
reader = yaml.NewReader(newReaderOptions(filePath, ""))
case "sqldir", "scripts", "scriptdir":
if filePath == "" {
return nil, fmt.Errorf("%s: file path is required for SQL directory format", label)
}
reader = sqldir.NewReader(&readers.ReaderOptions{FilePath: filePath})
reader = sqldir.NewReader(newReaderOptions(filePath, ""))
case "pgsql", "postgres", "postgresql":
if connString == "" {
return nil, fmt.Errorf("%s: connection string is required for PostgreSQL format", label)
}
reader = pgsql.NewReader(&readers.ReaderOptions{ConnectionString: connString})
reader = pgsql.NewReader(newReaderOptions("", connString))
case "sqlite", "sqlite3":
// SQLite can use either file path or connection string
@@ -276,7 +276,7 @@ func readDatabase(dbType, filePath, connString, label string) (*models.Database,
if dbPath == "" {
return nil, fmt.Errorf("%s: file path or connection string is required for SQLite format", label)
}
reader = sqlite.NewReader(&readers.ReaderOptions{FilePath: dbPath})
reader = sqlite.NewReader(newReaderOptions(dbPath, ""))
default:
return nil, fmt.Errorf("%s: unsupported database format: %s", label, dbType)
+158
View File
@@ -0,0 +1,158 @@
package main
import (
"bytes"
"os"
"path/filepath"
"strings"
"testing"
"github.com/spf13/cobra"
)
func newDryRunCmd() (*cobra.Command, *bytes.Buffer) {
var buf bytes.Buffer
cmd := &cobra.Command{}
cmd.SetOut(&buf)
return cmd, &buf
}
func TestRunConvert_DryRunWritesNothing(t *testing.T) {
defer func(a, b, c, d string, e bool) {
convertSourceType, convertSourcePath, convertTargetType, convertTargetPath, convertDryRun = a, b, c, d, e
}(convertSourceType, convertSourcePath, convertTargetType, convertTargetPath, convertDryRun)
dir := t.TempDir()
in := filepath.Join(dir, "in.json")
out := filepath.Join(dir, "out.json")
writeTestJSON(t, in, []string{"users", "posts"})
convertSourceType, convertSourcePath = "json", in
convertTargetType, convertTargetPath = "json", out
convertDryRun = true
cmd, buf := newDryRunCmd()
if err := runConvert(cmd, nil); err != nil {
t.Fatalf("dry run: %v", err)
}
if _, err := os.Stat(out); !os.IsNotExist(err) {
t.Fatal("dry run must not create the output file")
}
for _, want := range []string{"dry run", "users", "posts", out} {
if !strings.Contains(buf.String(), want) {
t.Errorf("plan missing %q:\n%s", want, buf.String())
}
}
// Normal behavior is unchanged.
convertDryRun = false
if err := runConvert(cmd, nil); err != nil {
t.Fatalf("real run: %v", err)
}
if _, err := os.Stat(out); err != nil {
t.Fatalf("real run should write output: %v", err)
}
}
func TestRunConvert_DryRunValidatesTarget(t *testing.T) {
defer func(a, b, c, d string, e bool) {
convertSourceType, convertSourcePath, convertTargetType, convertTargetPath, convertDryRun = a, b, c, d, e
}(convertSourceType, convertSourcePath, convertTargetType, convertTargetPath, convertDryRun)
dir := t.TempDir()
in := filepath.Join(dir, "in.json")
out := filepath.Join(dir, "models")
writeTestJSON(t, in, []string{"users"})
convertSourceType, convertSourcePath = "json", in
convertDryRun = true
// gorm without --package must fail validation, as a real run would.
convertTargetType, convertTargetPath = "gorm", out
cmd, _ := newDryRunCmd()
err := runConvert(cmd, nil)
if err == nil || !strings.Contains(err.Error(), "package name is required") {
t.Fatalf("expected package validation error, got %v", err)
}
convertTargetType = "nope"
err = runConvert(cmd, nil)
if err == nil || !strings.Contains(err.Error(), "unsupported target format") {
t.Fatalf("expected unsupported format error, got %v", err)
}
if _, statErr := os.Stat(out); !os.IsNotExist(statErr) {
t.Fatal("dry run must not create the output path")
}
}
func TestRunSplit_DryRunWritesNothing(t *testing.T) {
defer func(a, b, c, d, e string, f bool) {
splitSourceType, splitSourcePath, splitTargetType, splitTargetPath, splitTables, splitDryRun = a, b, c, d, e, f
}(splitSourceType, splitSourcePath, splitTargetType, splitTargetPath, splitTables, splitDryRun)
dir := t.TempDir()
in := filepath.Join(dir, "in.json")
out := filepath.Join(dir, "subset.json")
writeTestJSON(t, in, []string{"users", "posts", "comments"})
splitSourceType, splitSourcePath = "json", in
splitTargetType, splitTargetPath = "json", out
splitTables = "users,posts"
splitDryRun = true
cmd, buf := newDryRunCmd()
if err := runSplit(cmd, nil); err != nil {
t.Fatalf("dry run: %v", err)
}
if _, err := os.Stat(out); !os.IsNotExist(err) {
t.Fatal("dry run must not create the output file")
}
got := buf.String()
if !strings.Contains(got, "2 table(s)") || strings.Contains(got, "comments") {
t.Errorf("plan should show only the 2 selected tables:\n%s", got)
}
// A selection that matches nothing fails validation in dry-run too.
splitTables = "does_not_exist"
if err := runSplit(cmd, nil); err == nil {
t.Fatal("expected error for empty selection")
}
}
func TestRunMerge_DryRunWritesNothing(t *testing.T) {
saved := saveMergeState()
defer restoreMergeState(saved)
defer func(v bool) { mergeDryRun = v }(mergeDryRun)
dir := t.TempDir()
target := filepath.Join(dir, "target.json")
source := filepath.Join(dir, "source.json")
out := filepath.Join(dir, "merged.json")
writeTestJSON(t, target, []string{"users"})
writeTestJSON(t, source, []string{"posts"})
mergeTargetType, mergeTargetPath, mergeTargetConn = "json", target, ""
mergeSourceType, mergeSourcePath, mergeSourceConn = "json", source, ""
mergeFromList = nil
mergeOutputType, mergeOutputPath, mergeOutputConn = "json", out, ""
mergeSkipTables, mergeReportPath = "", ""
mergeDryRun = true
cmd, buf := newDryRunCmd()
if err := runMerge(cmd, nil); err != nil {
t.Fatalf("dry run: %v", err)
}
if _, err := os.Stat(out); !os.IsNotExist(err) {
t.Fatal("dry run must not create the output file")
}
for _, want := range []string{"dry run", "users", "posts", out} {
if !strings.Contains(buf.String(), want) {
t.Errorf("plan missing %q:\n%s", want, buf.String())
}
}
mergeOutputType = "nope"
if err := runMerge(cmd, nil); err == nil || !strings.Contains(err.Error(), "unsupported format") {
t.Fatalf("expected unsupported output format error, got %v", err)
}
}
+1 -1
View File
@@ -193,7 +193,7 @@ func runInspect(cmd *cobra.Command, args []string) error {
// Write output
if inspectOutputPath != "" {
err = os.WriteFile(inspectOutputPath, []byte(formattedReport), 0644)
err = os.WriteFile(inspectOutputPath, []byte(formattedReport), 0o644)
if err != nil {
return fmt.Errorf("failed to write output file: %w", err)
}
+427 -21
View File
@@ -1,6 +1,7 @@
package main
import (
"bytes"
"fmt"
"io"
"os"
@@ -11,12 +12,17 @@ import (
"github.com/spf13/cobra"
"git.warky.dev/wdevs/relspecgo/pkg/diff"
"git.warky.dev/wdevs/relspecgo/pkg/inspector"
"git.warky.dev/wdevs/relspecgo/pkg/jobs"
"git.warky.dev/wdevs/relspecgo/pkg/merge"
"git.warky.dev/wdevs/relspecgo/pkg/models"
"git.warky.dev/wdevs/relspecgo/pkg/readers"
"git.warky.dev/wdevs/relspecgo/pkg/readers/sqldir"
"git.warky.dev/wdevs/relspecgo/pkg/writers"
wpgsql "git.warky.dev/wdevs/relspecgo/pkg/writers/pgsql"
"git.warky.dev/wdevs/relspecgo/pkg/writers/sqlexec"
wtemplate "git.warky.dev/wdevs/relspecgo/pkg/writers/template"
)
var (
@@ -54,8 +60,9 @@ inputs, output and options:
logfile: .relspec/log/build-schema.log
Rules and guarantees:
- command is a closed allow-list (convert, merge, scripts-list). Arbitrary
shell strings are never executed.
- command is a closed allow-list (convert, merge, split, scripts-list,
scripts-exec, templ, inspect, diff). Arbitrary shell strings are never
executed.
- Every path is relative to the directory holding the job file and may not
escape it. Absolute and home-relative paths are rejected.
- Remote database credentials are referenced by environment-variable name
@@ -120,6 +127,9 @@ func loadJobSet() (*jobs.Set, error) {
if err := set.Validate(); err != nil {
return nil, err
}
for _, w := range set.Warnings {
fmt.Fprintf(os.Stderr, "warning: %s\n", w)
}
return set, nil
}
@@ -178,12 +188,14 @@ func executeJobPlan(set *jobs.Set, name string, dryRun, noDeps bool, out io.Writ
// Pre-flight: resolve and check paths, output policy and env vars for the
// whole plan before anything runs. A failure here means no job executes.
resolved := make([]*resolvedJob, len(plan))
byName := make(map[string]*resolvedJob, len(plan))
for i, j := range plan {
rj, perr := preflightJob(j)
rj, perr := preflightJob(j, byName)
if perr != nil {
return fmt.Errorf("job %q: %w", j.Name, perr)
}
resolved[i] = rj
byName[j.Name] = rj
}
if dryRun {
@@ -215,6 +227,12 @@ type resolvedJob struct {
outputConn string // resolved connection string (secret)
outputConnEnv string
logPath string
logPolicy jobs.LogPolicy
templatePath string
reportPath string // "" for a diff summary written to the log
reportFormat string
rulesPath string // "" means inspector defaults
selection *splitSelection
secrets []string // resolved secret values to redact from logs
}
@@ -223,22 +241,63 @@ type resolvedInput struct {
path string // "" when the input is a database
conn string // resolved connection string (secret)
connEnv string
fromJob string // producer job name when this input came from from_job
}
func preflightJob(j *jobs.Job) (*resolvedJob, error) {
func preflightJob(j *jobs.Job, resolvedByName map[string]*resolvedJob) (*resolvedJob, error) {
root := j.Dir()
rj := &resolvedJob{job: j, root: root}
rj := &resolvedJob{job: j, root: root, logPolicy: j.ResolvedLogPolicy()}
expandPath := func(label, value string) (string, error) {
expanded, err := jobs.ExpandEnv(value)
if err != nil {
return "", fmt.Errorf("%s: %w", label, err)
}
return expanded, nil
}
if j.Logfile != "" {
p, err := jobs.SafeJoin(root, j.Logfile)
logfile, err := expandPath("logfile", j.Logfile)
if err != nil {
return nil, err
}
p, err := jobs.SafeJoin(root, logfile)
if err != nil {
return nil, fmt.Errorf("logfile: %w", err)
}
rj.logPath = p
}
if j.Template != "" {
templatePath, err := expandPath("template", j.Template)
if err != nil {
return nil, err
}
p, err := jobs.SafeJoin(root, templatePath)
if err != nil {
return nil, fmt.Errorf("template: %w", err)
}
info, err := os.Stat(p)
if err != nil || info.IsDir() {
return nil, fmt.Errorf("template %q: not found or is a directory", templatePath)
}
rj.templatePath = p
}
for i, in := range j.Inputs {
ri := resolvedInput{format: strings.ToLower(in.Format)}
if in.FromJob != "" {
producer, ok := resolvedByName[in.FromJob]
if !ok {
return nil, fmt.Errorf("input[%d]: from_job %q is not in this plan (do not use --no-deps with from_job inputs)", i, in.FromJob)
}
if producer.outputPath == "" {
return nil, fmt.Errorf("input[%d]: from_job %q does not write a file output", i, in.FromJob)
}
ri.path = producer.outputPath
ri.format = strings.ToLower(producer.job.Output.Format)
ri.fromJob = in.FromJob
rj.inputs = append(rj.inputs, ri)
continue
}
if in.ConnEnv != "" {
v, ok := os.LookupEnv(in.ConnEnv)
if !ok || v == "" {
@@ -248,16 +307,20 @@ func preflightJob(j *jobs.Job) (*resolvedJob, error) {
ri.connEnv = in.ConnEnv
rj.secrets = append(rj.secrets, v)
} else {
p, err := jobs.SafeJoin(root, in.Path)
inputPath, err := expandPath(fmt.Sprintf("input[%d]", i), in.Path)
if err != nil {
return nil, err
}
p, err := jobs.SafeJoin(root, inputPath)
if err != nil {
return nil, fmt.Errorf("input[%d]: %w", i, err)
}
info, err := os.Stat(p)
if err != nil {
return nil, fmt.Errorf("input[%d]: %s: file not found", i, in.Path)
return nil, fmt.Errorf("input[%d]: %s: file not found", i, inputPath)
}
if info.IsDir() {
return nil, fmt.Errorf("input[%d]: %s: is a directory, not a file", i, in.Path)
return nil, fmt.Errorf("input[%d]: %s: is a directory, not a file", i, inputPath)
}
ri.path = p
}
@@ -265,13 +328,17 @@ func preflightJob(j *jobs.Job) (*resolvedJob, error) {
}
for _, d := range j.ScriptDirs {
p, err := jobs.SafeJoin(root, d)
scriptDir, err := expandPath("script_dir", d)
if err != nil {
return nil, err
}
p, err := jobs.SafeJoin(root, scriptDir)
if err != nil {
return nil, fmt.Errorf("script_dir %q: %w", d, err)
}
info, err := os.Stat(p)
if err != nil {
return nil, fmt.Errorf("script_dir %q: not found", d)
return nil, fmt.Errorf("script_dir %q: not found", scriptDir)
}
if !info.IsDir() {
return nil, fmt.Errorf("script_dir %q: not a directory", d)
@@ -289,16 +356,65 @@ func preflightJob(j *jobs.Job) (*resolvedJob, error) {
rj.outputConnEnv = j.Output.ConnEnv
rj.secrets = append(rj.secrets, v)
} else {
p, err := jobs.SafeJoin(root, j.Output.Path)
outputPath, err := expandPath("output", j.Output.Path)
if err != nil {
return nil, err
}
p, err := jobs.SafeJoin(root, outputPath)
if err != nil {
return nil, fmt.Errorf("output: %w", err)
}
if _, err := os.Stat(p); err == nil && !j.Output.Overwrite {
return nil, fmt.Errorf("output %s already exists (set output.overwrite: true to replace it)", j.Output.Path)
return nil, fmt.Errorf("output %s already exists (set output.overwrite: true to replace it)", outputPath)
}
rj.outputPath = p
}
}
if j.Rules != "" {
rulesPath, err := expandPath("rules", j.Rules)
if err != nil {
return nil, err
}
p, err := jobs.SafeJoin(root, rulesPath)
if err != nil {
return nil, fmt.Errorf("rules: %w", err)
}
info, err := os.Stat(p)
if err != nil || info.IsDir() {
return nil, fmt.Errorf("rules %q: not found or is a directory", rulesPath)
}
rj.rulesPath = p
}
if j.Report != nil {
rj.reportFormat = strings.ToLower(j.Report.Format)
if j.Report.Path != "" {
reportPath, err := expandPath("report", j.Report.Path)
if err != nil {
return nil, err
}
p, err := jobs.SafeJoin(root, reportPath)
if err != nil {
return nil, fmt.Errorf("report: %w", err)
}
if _, err := os.Stat(p); err == nil && !j.Report.Overwrite {
return nil, fmt.Errorf("report %s already exists (set report.overwrite: true to replace it)", reportPath)
}
rj.reportPath = p
}
}
if j.Select != nil {
rj.selection = &splitSelection{
Schemas: j.Select.Schemas,
Tables: j.Select.Tables,
ExcludeSchemas: j.Select.ExcludeSchemas,
ExcludeTables: j.Select.ExcludeTables,
DatabaseName: j.Select.DatabaseName,
}
}
return rj, nil
}
@@ -311,9 +427,12 @@ func printResolvedJob(out io.Writer, n, total int, rj *resolvedJob) {
}
fmt.Fprintf(out, " job file: %s\n", j.SourceFile)
for _, ri := range rj.inputs {
if ri.path != "" {
switch {
case ri.fromJob != "":
fmt.Fprintf(out, " input: %s (%s) from job %q\n", ri.path, ri.format, ri.fromJob)
case ri.path != "":
fmt.Fprintf(out, " input: %s (%s)\n", ri.path, ri.format)
} else {
default:
fmt.Fprintf(out, " input: env:%s (%s)\n", ri.connEnv, ri.format)
}
}
@@ -325,15 +444,31 @@ func printResolvedJob(out io.Writer, n, total int, rj *resolvedJob) {
} else if rj.outputConnEnv != "" {
fmt.Fprintf(out, " output: env:%s (%s)\n", rj.outputConnEnv, j.Output.Format)
}
if j.Report != nil {
format := valueOr(rj.reportFormat, "default")
if rj.reportPath != "" {
fmt.Fprintf(out, " report: %s (%s)\n", rj.reportPath, format)
} else {
fmt.Fprintf(out, " report: (log) (%s)\n", format)
}
}
if rj.rulesPath != "" {
fmt.Fprintf(out, " rules: %s\n", rj.rulesPath)
} else if j.Command == jobs.CommandInspect {
fmt.Fprintf(out, " rules: (built-in defaults)\n")
}
if rj.selection != nil {
fmt.Fprintf(out, " select: %s\n", rj.selection.summary())
}
if rj.logPath != "" {
fmt.Fprintf(out, " logfile: %s\n", rj.logPath)
fmt.Fprintf(out, " logfile: %s (rotate >= %d bytes, keep %d)\n", rj.logPath, rj.logPolicy.MaxSizeBytes, rj.logPolicy.Keep)
}
fmt.Fprintln(out)
}
// executeResolvedJob runs a single already-validated job.
func executeResolvedJob(rj *resolvedJob) (err error) {
lg, closeLog, lerr := newJobLogger(rj.logPath, rj.secrets)
lg, closeLog, lerr := newJobLogger(rj.logPath, rj.logPolicy, rj.secrets)
if lerr != nil {
return lerr
}
@@ -348,6 +483,16 @@ func executeResolvedJob(rj *resolvedJob) (err error) {
err = runMergeJob(rj, lg)
case jobs.CommandScriptsList:
err = runScriptsListJob(rj, lg)
case jobs.CommandTempl:
err = runTemplJob(rj, lg)
case jobs.CommandSplit:
err = runSplitJob(rj, lg)
case jobs.CommandInspect:
err = runInspectJob(rj, lg)
case jobs.CommandDiff:
err = runDiffJob(rj, lg)
case jobs.CommandScriptsExec:
err = runScriptsExecJob(rj, lg)
default:
err = fmt.Errorf("unsupported command %q", rj.job.Command)
}
@@ -359,6 +504,50 @@ func executeResolvedJob(rj *resolvedJob) (err error) {
return err
}
func runTemplJob(rj *resolvedJob, lg *jobLogger) error {
db, err := readJobInputs(rj, lg)
if err != nil {
return err
}
if schema := rj.job.Options.Schema; schema != "" {
found := false
for _, s := range db.Schemas {
if s.Name == schema {
db.Schemas = []*models.Schema{s}
found = true
break
}
}
if !found {
return fmt.Errorf("schema not found: %s", schema)
}
}
mode := rj.job.Mode
if mode == "" {
mode = "database"
}
pattern := rj.job.FilenamePattern
if pattern == "" {
pattern = "{{.Name}}.txt"
}
writer, err := wtemplate.NewWriter(&writers.WriterOptions{
OutputPath: rj.outputPath,
Metadata: map[string]interface{}{
"template_path": rj.templatePath,
"mode": mode,
"filename_pattern": pattern,
},
})
if err != nil {
return fmt.Errorf("create template writer: %w", err)
}
lg.logf("applying template: %s (mode %s)", rj.templatePath, mode)
if err := writer.WriteDatabase(db); err != nil {
return fmt.Errorf("execute template: %w", err)
}
return nil
}
func runConvertJob(rj *resolvedJob, lg *jobLogger) error {
db, err := readJobInputs(rj, lg)
if err != nil {
@@ -389,6 +578,7 @@ func runMergeJob(rj *resolvedJob, lg *jobLogger) error {
lg.logf("merging: %s", inputLabel(ri))
merge.MergeDatabases(base, db, opts)
}
finalizeCommentedRefs(base, func(w string) { lg.logf("warning: %s", w) })
base.UpdateDate()
return writeJobOutput(rj, base, lg)
}
@@ -445,6 +635,168 @@ func runScriptsListJob(rj *resolvedJob, lg *jobLogger) error {
return nil
}
func runSplitJob(rj *resolvedJob, lg *jobLogger) error {
db, err := readJobInputs(rj, lg)
if err != nil {
return err
}
sel := splitSelection{}
if rj.selection != nil {
sel = *rj.selection
}
filtered, err := filterDatabaseSelection(db, sel)
if err != nil {
return fmt.Errorf("split selection: %w", err)
}
if sel.DatabaseName != "" {
filtered.Name = sel.DatabaseName
}
tables := 0
for _, s := range filtered.Schemas {
tables += len(s.Tables)
}
lg.logf("split: selected %d schema(s), %d table(s)", len(filtered.Schemas), tables)
return writeJobOutput(rj, filtered, lg)
}
func runInspectJob(rj *resolvedJob, lg *jobLogger) error {
db, err := readJobInputs(rj, lg)
if err != nil {
return err
}
config, err := inspector.LoadConfig(rj.rulesPath) // "" -> built-in defaults
if err != nil {
return fmt.Errorf("load rules: %w", err)
}
report, err := inspector.NewInspector(db, config).Inspect()
if err != nil {
return fmt.Errorf("inspection failed: %w", err)
}
var formatted string
switch valueOr(rj.reportFormat, "markdown") {
case "json":
formatted, err = inspector.NewJSONFormatter().Format(report)
default:
formatted, err = inspector.NewMarkdownFormatter(io.Discard).Format(report)
}
if err != nil {
return fmt.Errorf("format report: %w", err)
}
if werr := atomicWrite(rj.reportPath, func(tmp string) error {
return os.WriteFile(tmp, []byte(formatted), 0o644)
}); werr != nil {
return werr
}
lg.logf("inspect: %d error(s), %d warning(s) -> %s",
report.Summary.ErrorCount, report.Summary.WarningCount, rj.reportPath)
if report.HasErrors() {
return fmt.Errorf("inspection found %d error(s)", report.Summary.ErrorCount)
}
return nil
}
func runDiffJob(rj *resolvedJob, lg *jobLogger) error {
if len(rj.inputs) != 2 {
return fmt.Errorf("diff requires exactly 2 inputs, got %d", len(rj.inputs))
}
source, err := readOneJobInput(rj.inputs[0])
if err != nil {
return fmt.Errorf("input[0]: %w", err)
}
lg.logf("diff source: %s", inputLabel(rj.inputs[0]))
target, err := readOneJobInput(rj.inputs[1])
if err != nil {
return fmt.Errorf("input[1]: %w", err)
}
lg.logf("diff target: %s", inputLabel(rj.inputs[1]))
result := diff.CompareDatabases(source, target)
s := diff.ComputeSummary(result)
lg.logf("diff: schemas %d/%d/%d, tables %d/%d/%d, columns %d/%d/%d (missing/extra/modified)",
s.Schemas.Missing, s.Schemas.Extra, s.Schemas.Modified,
s.Tables.Missing, s.Tables.Extra, s.Tables.Modified,
s.Columns.Missing, s.Columns.Extra, s.Columns.Modified)
format := diff.FormatSummary
switch rj.reportFormat {
case "json":
format = diff.FormatJSON
case "html":
format = diff.FormatHTML
}
if rj.reportPath == "" {
var buf bytes.Buffer
if err := diff.FormatDiff(result, format, &buf); err != nil {
return fmt.Errorf("format diff: %w", err)
}
for _, line := range strings.Split(strings.TrimRight(buf.String(), "\n"), "\n") {
lg.logf("%s", line)
}
return nil
}
if werr := atomicWrite(rj.reportPath, func(tmp string) error {
f, err := os.Create(tmp)
if err != nil {
return err
}
defer f.Close()
return diff.FormatDiff(result, format, f)
}); werr != nil {
return werr
}
lg.logf("diff report written: %s", rj.reportPath)
return nil
}
func runScriptsExecJob(rj *resolvedJob, lg *jobLogger) error {
schemaName := valueOr(rj.job.Options.Schema, "public")
combined := &models.Schema{Name: schemaName}
for _, dir := range rj.scriptDirs {
reader := sqldir.NewReader(&readers.ReaderOptions{
FilePath: dir,
Metadata: map[string]any{
"schema_name": schemaName,
"database_name": "database",
},
})
db, err := reader.ReadDatabase()
if err != nil {
return fmt.Errorf("%s: %w", dir, err)
}
if len(db.Schemas) == 0 {
continue
}
combined.Scripts = append(combined.Scripts, db.Schemas[0].Scripts...)
}
if len(combined.Scripts) == 0 {
lg.logf("no scripts found; nothing to execute")
return nil
}
lg.logf("executing %d script(s) against database env:%s", len(combined.Scripts), rj.outputConnEnv)
writer := sqlexec.NewWriter(&writers.WriterOptions{
Metadata: map[string]any{
"connection_string": rj.outputConn,
"ignore_errors": rj.job.Options.ContinueOnError,
},
})
if err := writer.WriteSchema(combined); err != nil {
return fmt.Errorf("script execution failed: %w", err)
}
opts := writer.Options()
total, _ := opts.Metadata["execution_total"].(int)
success, _ := opts.Metadata["execution_success"].(int)
failed, _ := opts.Metadata["execution_failed"].(int)
lg.logf("executed %d script(s): %d succeeded, %d failed", total, success, failed)
if failed > 0 && !rj.job.Options.ContinueOnError {
return fmt.Errorf("%d script(s) failed", failed)
}
return nil
}
// readJobInputs reads every input and additively merges them into one model.
func readJobInputs(rj *resolvedJob, lg *jobLogger) (*models.Database, error) {
var base *models.Database
@@ -463,6 +815,7 @@ func readJobInputs(rj *resolvedJob, lg *jobLogger) (*models.Database, error) {
if base == nil {
return nil, fmt.Errorf("no inputs produced a database")
}
finalizeCommentedRefs(base, func(w string) { lg.logf("warning: %s", w) })
return base, nil
}
@@ -490,8 +843,8 @@ func writeJobOutput(rj *resolvedJob, db *models.Database, lg *jobLogger) error {
return fmt.Errorf("database output is only supported for pgsql (got %q)", rj.job.Output.Format)
}
lg.logf("writing output to database env:%s", rj.outputConnEnv)
writerOpts := newWriterOptions("", o.Package, o.FlattenSchema, "", "", o.ContinueOnError)
writerOpts.Metadata = map[string]interface{}{"connection_string": rj.outputConn}
writerOpts := newWriterOptions("", o.Package, o.FlattenSchema, o.Types, o.ArrayNullable, o.ContinueOnError)
writerOpts.Metadata = map[string]interface{}{"connection_string": rj.outputConn, "full_ddl": o.FullDDL}
return wpgsql.NewWriter(writerOpts).WriteDatabase(db)
}
@@ -499,7 +852,37 @@ func writeJobOutput(rj *resolvedJob, db *models.Database, lg *jobLogger) error {
return fmt.Errorf("failed to create output directory: %w", err)
}
lg.logf("writing output: %s (%s)", rj.outputPath, format)
return writeDatabase(db, format, rj.outputPath, o.Package, o.Schema, o.FlattenSchema, "", "", o.ContinueOnError, "")
write := func(target string) error {
return writeDatabase(db, format, target, o.Package, o.Schema, o.FlattenSchema, o.Types, o.ArrayNullable, o.ContinueOnError, "")
}
// Single-file formats are written to a temp file and renamed into place so
// a failure never leaves a partial or truncated output. Directory-emitting
// formats (gorm/bun/drizzle/typeorm/prisma) write in place.
if jobs.SingleFileOutputFormat(format) {
return atomicWrite(rj.outputPath, write)
}
return write(rj.outputPath)
}
// atomicWrite calls produce with a temp path in the same directory as
// finalPath, then renames it over finalPath. The temp file is removed on any
// error so the destination is only ever replaced by a complete file.
func atomicWrite(finalPath string, produce func(tmpPath string) error) error {
dir := filepath.Dir(finalPath)
if err := os.MkdirAll(dir, 0o755); err != nil {
return fmt.Errorf("failed to create output directory: %w", err)
}
tmp := filepath.Join(dir, fmt.Sprintf(".%s.relspec-tmp-%d", filepath.Base(finalPath), os.Getpid()))
if err := produce(tmp); err != nil {
_ = os.Remove(tmp)
return err
}
if err := os.Rename(tmp, finalPath); err != nil {
_ = os.Remove(tmp)
return fmt.Errorf("failed to finalize %s: %w", finalPath, err)
}
return nil
}
// --- logging + redaction ---------------------------------------------------
@@ -512,7 +895,7 @@ type jobLogger struct {
// newJobLogger returns a logger that mirrors to stderr and, when path is set,
// to a job logfile. Connection strings and known secret values are redacted
// from everything it writes.
func newJobLogger(path string, secrets []string) (*jobLogger, func(err error), error) {
func newJobLogger(path string, policy jobs.LogPolicy, secrets []string) (*jobLogger, func(err error), error) {
lg := &jobLogger{secrets: secrets}
if path == "" {
return lg, func(error) {}, nil
@@ -520,6 +903,7 @@ func newJobLogger(path string, secrets []string) (*jobLogger, func(err error), e
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
return nil, nil, fmt.Errorf("failed to create log directory: %w", err)
}
rotateLogIfNeeded(path, policy)
f, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0o644)
if err != nil {
return nil, nil, fmt.Errorf("failed to open logfile %q: %w", path, err)
@@ -533,6 +917,28 @@ func newJobLogger(path string, secrets []string) (*jobLogger, func(err error), e
}, nil
}
// rotateLogIfNeeded renames path -> path.1 -> path.2 ... up to policy.Keep
// when path has grown to policy.MaxSizeBytes or more. The oldest file beyond
// Keep is deleted. A zero/negative MaxSizeBytes disables rotation.
func rotateLogIfNeeded(path string, policy jobs.LogPolicy) {
if policy.MaxSizeBytes <= 0 {
return
}
info, err := os.Stat(path)
if err != nil || info.Size() < policy.MaxSizeBytes {
return
}
if policy.Keep < 1 {
_ = os.Remove(path)
return
}
_ = os.Remove(fmt.Sprintf("%s.%d", path, policy.Keep))
for i := policy.Keep - 1; i >= 1; i-- {
_ = os.Rename(fmt.Sprintf("%s.%d", path, i), fmt.Sprintf("%s.%d", path, i+1))
}
_ = os.Rename(path, path+".1")
}
func (l *jobLogger) logf(format string, args ...interface{}) {
line := l.redact(fmt.Sprintf(format, args...))
fmt.Fprintf(os.Stderr, " %s\n", line)
+366
View File
@@ -375,3 +375,369 @@ jobs:
t.Fatal("job list output not deterministic")
}
}
func TestJobRun_SplitJob(t *testing.T) {
dir := jobFixture(t, `version: 1
jobs:
extract:
command: split
inputs:
- path: schema/core.dbml
format: dbml
- path: schema/tenant.dbml
format: dbml
select:
tables: [users]
output:
format: json
path: build/subset.json
overwrite: true
`)
set := mustLoadSet(t, filepath.Join(dir, "relspec.yml"))
if err := executeJobPlan(set, "extract", false, false, &bytes.Buffer{}); err != nil {
t.Fatalf("execute split job: %v", err)
}
out, err := os.ReadFile(filepath.Join(dir, "build", "subset.json"))
if err != nil {
t.Fatalf("read split output: %v", err)
}
s := string(out)
if !strings.Contains(s, "users") {
t.Fatalf("split output missing selected table:\n%s", s)
}
if strings.Contains(s, "posts") {
t.Fatalf("split output should have excluded posts:\n%s", s)
}
}
func TestJobRun_InspectJob(t *testing.T) {
dir := jobFixture(t, `version: 1
jobs:
lint:
command: inspect
inputs:
- path: schema/core.dbml
format: dbml
report:
format: json
path: build/report.json
overwrite: true
logfile: .relspec/lint.log
`)
set := mustLoadSet(t, filepath.Join(dir, "relspec.yml"))
// Default rules only warn, so the job succeeds.
if err := executeJobPlan(set, "lint", false, false, &bytes.Buffer{}); err != nil {
t.Fatalf("execute inspect job: %v", err)
}
if _, err := os.ReadFile(filepath.Join(dir, "build", "report.json")); err != nil {
t.Fatalf("expected report file: %v", err)
}
logData, _ := os.ReadFile(filepath.Join(dir, ".relspec", "lint.log"))
if !strings.Contains(string(logData), "inspect:") {
t.Fatalf("logfile missing inspect summary:\n%s", logData)
}
}
func TestJobRun_InspectJobFailsOnRuleError(t *testing.T) {
dir := jobFixture(t, `version: 1
jobs:
lint:
command: inspect
inputs:
- path: schema/core.dbml
format: dbml
rules: rules.yaml
report:
format: json
path: build/report.json
overwrite: true
logfile: .relspec/lint.log
`)
// A rule set to "error" level for a violation the fixture triggers.
writeFile(t, filepath.Join(dir, "rules.yaml"), `version: "1.0"
rules:
primary_key_naming:
enabled: enforce
function: primary_key_naming
pattern: "^id_"
message: "Primary key columns should start with id_"
`)
set := mustLoadSet(t, filepath.Join(dir, "relspec.yml"))
err := executeJobPlan(set, "lint", false, false, &bytes.Buffer{})
if err == nil || !strings.Contains(err.Error(), "error(s)") {
t.Fatalf("expected inspect job to fail on rule error, got %v", err)
}
logData, _ := os.ReadFile(filepath.Join(dir, ".relspec", "lint.log"))
if !strings.Contains(string(logData), "FAILED") {
t.Fatalf("failed inspect job should log FAILED:\n%s", logData)
}
}
func TestJobRun_DiffJob(t *testing.T) {
dir := jobFixture(t, `version: 1
jobs:
compare:
command: diff
inputs:
- path: schema/core.dbml
format: dbml
- path: schema/tenant.dbml
format: dbml
report:
format: json
path: build/diff.json
overwrite: true
`)
set := mustLoadSet(t, filepath.Join(dir, "relspec.yml"))
if err := executeJobPlan(set, "compare", false, false, &bytes.Buffer{}); err != nil {
t.Fatalf("execute diff job: %v", err)
}
out, err := os.ReadFile(filepath.Join(dir, "build", "diff.json"))
if err != nil {
t.Fatalf("read diff report: %v", err)
}
if len(out) == 0 {
t.Fatal("diff report is empty")
}
}
func TestJobRun_FromJobWiring(t *testing.T) {
dir := jobFixture(t, `version: 1
jobs:
a:
command: convert
inputs:
- path: schema/core.dbml
format: dbml
output:
format: json
path: build/a.json
overwrite: true
b:
command: convert
inputs:
- from_job: a
output:
format: yaml
path: build/b.yaml
overwrite: true
`)
set := mustLoadSet(t, filepath.Join(dir, "relspec.yml"))
if err := executeJobPlan(set, "b", false, false, &bytes.Buffer{}); err != nil {
t.Fatalf("execute from_job chain: %v", err)
}
if _, err := os.Stat(filepath.Join(dir, "build", "a.json")); err != nil {
t.Fatalf("producer output missing: %v", err)
}
out, err := os.ReadFile(filepath.Join(dir, "build", "b.yaml"))
if err != nil {
t.Fatalf("consumer output missing: %v", err)
}
if !strings.Contains(string(out), "users") {
t.Fatalf("consumer did not consume producer output:\n%s", out)
}
}
func TestJobRun_LogRotation(t *testing.T) {
dir := jobFixture(t, `version: 1
jobs:
build:
command: scripts-list
script_dirs: [migrations]
log_max_size: "150B"
log_keep: 2
logfile: .relspec/build.log
`)
writeFile(t, filepath.Join(dir, "migrations", "1_001_a.sql"), "CREATE TABLE a();\n")
logPath := filepath.Join(dir, ".relspec", "build.log")
writeFile(t, logPath, strings.Repeat("x", 300)+"\n")
set := mustLoadSet(t, filepath.Join(dir, "relspec.yml"))
if err := executeJobPlan(set, "build", false, false, &bytes.Buffer{}); err != nil {
t.Fatalf("execute job: %v", err)
}
rotated, err := os.ReadFile(logPath + ".1")
if err != nil {
t.Fatalf("expected rotated logfile build.log.1: %v", err)
}
if !strings.Contains(string(rotated), strings.Repeat("x", 300)) {
t.Fatalf("rotated logfile should hold the old content")
}
fresh, err := os.ReadFile(logPath)
if err != nil {
t.Fatalf("expected fresh logfile: %v", err)
}
if strings.Contains(string(fresh), strings.Repeat("x", 300)) {
t.Fatalf("fresh logfile should not contain the rotated-out content:\n%s", fresh)
}
if !strings.Contains(string(fresh), "OK") {
t.Fatalf("fresh logfile should hold the new run:\n%s", fresh)
}
}
func TestJobRun_AtomicOutputLeavesOriginalOnFailure(t *testing.T) {
dir := jobFixture(t, `version: 1
jobs:
x:
command: convert
inputs:
- path: schema/core.dbml
format: dbml
output:
format: json
path: build/out.json
overwrite: true
`)
// Seed the destination, then make its parent directory read-only so the
// rename step fails. The seeded file must survive intact.
seeded := filepath.Join(dir, "build", "out.json")
writeFile(t, seeded, `{"seeded":true}`)
if err := os.Chmod(filepath.Join(dir, "build"), 0o500); err != nil {
t.Skipf("cannot chmod: %v", err)
}
t.Cleanup(func() { _ = os.Chmod(filepath.Join(dir, "build"), 0o755) })
set := mustLoadSet(t, filepath.Join(dir, "relspec.yml"))
if err := executeJobPlan(set, "x", false, false, &bytes.Buffer{}); err == nil {
t.Skip("write unexpectedly succeeded (running as root?)")
}
if err := os.Chmod(filepath.Join(dir, "build"), 0o755); err != nil {
t.Fatal(err)
}
data, err := os.ReadFile(seeded)
if err != nil {
t.Fatalf("seeded file gone: %v", err)
}
if !strings.Contains(string(data), "seeded") {
t.Fatalf("seeded file was corrupted: %s", data)
}
}
func TestJobRun_ScriptsExecMissingConnEnv(t *testing.T) {
dir := jobFixture(t, `version: 1
jobs:
migrate:
command: scripts-exec
script_dirs: [migrations]
output:
conn_env: RELSPEC_TEST_EXEC_MISSING
logfile: .relspec/migrate.log
`)
writeFile(t, filepath.Join(dir, "migrations", "1_001_a.sql"), "CREATE TABLE a();\n")
os.Unsetenv("RELSPEC_TEST_EXEC_MISSING")
set := mustLoadSet(t, filepath.Join(dir, "relspec.yml"))
err := executeJobPlan(set, "migrate", false, false, &bytes.Buffer{})
if err == nil || !strings.Contains(err.Error(), "conn_env") {
t.Fatalf("expected missing conn_env error, got %v", err)
}
}
func TestJobRun_ScriptsExecDryRun(t *testing.T) {
dir := jobFixture(t, `version: 1
jobs:
migrate:
command: scripts-exec
script_dirs: [migrations]
output:
conn_env: RELSPEC_TEST_EXEC_CONN
`)
writeFile(t, filepath.Join(dir, "migrations", "1_001_a.sql"), "CREATE TABLE a();\n")
t.Setenv("RELSPEC_TEST_EXEC_CONN", "postgres://u:secretpw@h/db")
set := mustLoadSet(t, filepath.Join(dir, "relspec.yml"))
var buf bytes.Buffer
if err := executeJobPlan(set, "migrate", true, false, &buf); err != nil {
t.Fatalf("dry run: %v", err)
}
if strings.Contains(buf.String(), "secretpw") {
t.Fatalf("plan leaked secret:\n%s", buf.String())
}
if !strings.Contains(buf.String(), "env:RELSPEC_TEST_EXEC_CONN") {
t.Fatalf("plan should name the env var:\n%s", buf.String())
}
}
func TestJobRun_TemplDatabaseMode(t *testing.T) {
dir := jobFixture(t, `version: 1
jobs:
docs:
command: templ
inputs:
- path: schema/core.dbml
format: dbml
template: templates/schema.tmpl
output:
format: text
path: build/schema.txt
overwrite: true
`)
writeFile(t, filepath.Join(dir, "templates", "schema.tmpl"), "{{range .Database.Schemas}}{{range .Tables}}{{.Name}} {{end}}{{end}}")
set := mustLoadSet(t, filepath.Join(dir, "relspec.yml"))
if err := executeJobPlan(set, "docs", false, false, &bytes.Buffer{}); err != nil {
t.Fatalf("execute templ job: %v", err)
}
out, err := os.ReadFile(filepath.Join(dir, "build", "schema.txt"))
if err != nil {
t.Fatalf("read templ output: %v", err)
}
if !strings.Contains(string(out), "users") {
t.Fatalf("templ output missing users table: %s", out)
}
}
func TestJobRun_BunSqlTypesOption(t *testing.T) {
dir := jobFixture(t, `version: 1
jobs:
models:
command: convert
inputs:
- path: schema/core.dbml
format: dbml
output:
format: bun
path: build/models
options:
package: models
types: sqltypes
`)
set := mustLoadSet(t, filepath.Join(dir, "relspec.yml"))
if err := executeJobPlan(set, "models", false, false, &bytes.Buffer{}); err != nil {
t.Fatalf("execute Bun job: %v", err)
}
generated, err := os.ReadFile(filepath.Join(dir, "build", "models"))
if err != nil {
t.Fatalf("read Bun output: %v", err)
}
if !strings.Contains(string(generated), "git.warky.dev/wdevs/relspecgo/pkg/sqltypes") {
t.Fatalf("Bun output did not preserve options.types: %s", generated)
}
}
func TestJobValidation_TemplTextFormat(t *testing.T) {
validDir := jobFixture(t, `version: 1
jobs:
docs:
command: templ
inputs: [{path: schema/core.dbml, format: dbml}]
template: schema.tmpl
output: {format: text, path: build/docs.txt}
`)
valid := mustLoadSet(t, filepath.Join(validDir, "relspec.yml"))
if err := valid.Validate(); err != nil {
t.Fatalf("text output should be valid for templ: %v", err)
}
invalidDir := jobFixture(t, `version: 1
jobs:
docs:
command: templ
inputs: [{path: schema/core.dbml, format: dbml}]
template: schema.tmpl
output: {format: json, path: build/docs.txt}
`)
invalid, err := jobs.Load([]string{filepath.Join(invalidDir, "relspec.yml")})
if err != nil {
t.Fatalf("load invalid manifest: %v", err)
}
if err := invalid.Validate(); err == nil || !strings.Contains(err.Error(), "output.format: text") {
t.Fatalf("expected templ format rejection, got %v", err)
}
}
+35 -2
View File
@@ -5,10 +5,43 @@ import (
"os"
)
// asciiLogo (see version.go) is printed by the `version` command.
func main() {
printVersionHeader(os.Args[1:])
if err := rootCmd.Execute(); err != nil {
args := os.Args[1:]
isSilent := hasSilentFlag(args)
if !isSilent {
printVersionHeader(args)
}
previousStderr := os.Stderr
var nullOutput *os.File
if isSilent {
var err error
nullOutput, err = os.OpenFile(os.DevNull, os.O_WRONLY, 0)
if err != nil {
fmt.Fprintln(previousStderr, err)
os.Exit(1)
}
os.Stderr = nullOutput
}
err := rootCmd.Execute()
if nullOutput != nil {
os.Stderr = previousStderr
nullOutput.Close()
}
if err != nil {
fmt.Fprintln(os.Stderr, err)
os.Exit(1)
}
}
func hasSilentFlag(args []string) bool {
for _, arg := range args {
if arg == "--silent" || arg == "--silent=true" {
return true
}
}
return false
}
+45 -5
View File
@@ -59,7 +59,9 @@ var (
mergeSkipTables string // Comma-separated table names to skip
mergeVerbose bool
mergeReportPath string // Path to write merge report
mergeFullDDL bool // Execute full DDL instead of diffing the live pgsql database
mergeFlattenSchema bool
mergeDryRun bool
)
var mergeCmd = &cobra.Command{
@@ -127,7 +129,9 @@ func init() {
mergeCmd.Flags().BoolVar(&mergeSkipSequences, "skip-sequences", false, "Skip sequences during merge")
mergeCmd.Flags().StringVar(&mergeSkipTables, "skip-tables", "", "Comma-separated list of table names to skip during merge")
mergeCmd.Flags().BoolVar(&mergeVerbose, "verbose", false, "Show verbose output")
mergeCmd.Flags().BoolVar(&mergeFullDDL, "full-ddl", false, "pgsql database output: execute the full DDL instead of only the differences from the live database")
mergeCmd.Flags().StringVar(&mergeReportPath, "merge-report", "", "Path to write merge report (JSON format)")
mergeCmd.Flags().BoolVar(&mergeDryRun, "dry-run", false, "Read and merge in memory, then print the merge plan without writing any output")
mergeCmd.Flags().BoolVar(&mergeFlattenSchema, "flatten-schema", false, "Flatten schema.table names to schema_table (useful for databases like SQLite that do not support schemas)")
}
@@ -158,9 +162,7 @@ func runMerge(cmd *cobra.Command, args []string) error {
}
mergeTargetPath = expandPath(mergeTargetPath)
} else if mergeTargetConn == "" {
return fmt.Errorf("--target-conn is required for pgsql format")
}
if mergeSourceType != "pgsql" {
@@ -180,7 +182,7 @@ func runMerge(cmd *cobra.Command, args []string) error {
}
// Step 1: Read target database
fmt.Fprintf(os.Stderr, "[1/3] Reading target database...\n")
fmt.Fprintf(os.Stderr, "[1/4] Reading target database...\n")
fmt.Fprintf(os.Stderr, " Format: %s\n", mergeTargetType)
if mergeTargetPath != "" {
fmt.Fprintf(os.Stderr, " Path: %s\n", mergeTargetPath)
@@ -197,7 +199,7 @@ func runMerge(cmd *cobra.Command, args []string) error {
printDatabaseStats(targetDB)
// Step 2: Read source database(s)
fmt.Fprintf(os.Stderr, "\n[2/3] Reading source database...\n")
fmt.Fprintf(os.Stderr, "\n[2/4] Reading source database...\n")
fmt.Fprintf(os.Stderr, " Format: %s\n", mergeSourceType)
var sourceDB *models.Database
@@ -231,7 +233,7 @@ func runMerge(cmd *cobra.Command, args []string) error {
printDatabaseStats(sourceDB)
// Step 3: Merge databases
fmt.Fprintf(os.Stderr, "\n[3/3] Merging databases...\n")
fmt.Fprintf(os.Stderr, "\n[3/4] Merging databases...\n")
opts := &merge.MergeOptions{
SkipDomains: mergeSkipDomains,
@@ -250,6 +252,7 @@ func runMerge(cmd *cobra.Command, args []string) error {
}
result := merge.MergeDatabases(targetDB, sourceDB, opts)
finalizeCommentedRefs(targetDB, stderrWarn)
// Update timestamp
targetDB.UpdateDate()
@@ -263,12 +266,38 @@ func runMerge(cmd *cobra.Command, args []string) error {
merge.GetColumnTypeConflictSummary(result, 10))
}
if mergeDryRun {
if !isMergeOutputFormat(mergeOutputType) {
return fmt.Errorf("dry run validation failed: Output: unsupported format '%s'", mergeOutputType)
}
out := outWriter(cmd)
fmt.Fprintf(out, "RelSpec merge plan (dry run - nothing written):\n")
fmt.Fprintf(out, " Target: %s database '%s'\n", mergeTargetType, targetDB.Name)
fmt.Fprintf(out, " Source: %s database '%s'\n", mergeSourceType, sourceDB.Name)
switch {
case mergeOutputPath != "":
fmt.Fprintf(out, " Output: %s -> %s\n", mergeOutputType, mergeOutputPath)
case mergeOutputConn != "":
fmt.Fprintf(out, " Output: %s -> %s\n", mergeOutputType, maskPassword(mergeOutputConn))
default:
fmt.Fprintf(out, " Output: %s\n", mergeOutputType)
}
fmt.Fprintf(out, " Result:\n")
printDryRunPlan(out, targetDB)
fmt.Fprintf(out, "\n%s\n", merge.GetMergeSummary(result))
fmt.Fprintf(os.Stderr, "\n=== Dry run complete: no output written ===\n")
return nil
}
// Step 4: Write output
fmt.Fprintf(os.Stderr, "\n[4/4] Writing output...\n")
fmt.Fprintf(os.Stderr, " Format: %s\n", mergeOutputType)
if mergeOutputPath != "" {
fmt.Fprintf(os.Stderr, " Path: %s\n", mergeOutputPath)
}
if mergeOutputConn != "" {
fmt.Fprintf(os.Stderr, " Conn: %s\n", maskPassword(mergeOutputConn))
}
err = writeDatabaseForMerge(mergeOutputType, mergeOutputPath, mergeOutputConn, targetDB, "Output", mergeFlattenSchema)
if err != nil {
@@ -281,6 +310,16 @@ func runMerge(cmd *cobra.Command, args []string) error {
return nil
}
// isMergeOutputFormat reports whether writeDatabaseForMerge supports dbType.
func isMergeOutputFormat(dbType string) bool {
switch strings.ToLower(dbType) {
case "dbml", "dctx", "drawdb", "graphql", "json", "yaml", "gorm", "bun",
"drizzle", "prisma", "typeorm", "sqlite", "sqlite3", "pgsql":
return true
}
return false
}
func readDatabaseForMerge(dbType, filePath, connString, label string) (*models.Database, error) {
var reader readers.Reader
@@ -441,6 +480,7 @@ func writeDatabaseForMerge(dbType, filePath, connString string, db *models.Datab
if connString != "" {
writerOpts.Metadata = map[string]interface{}{
"connection_string": connString,
"full_ddl": mergeFullDDL,
}
// Add report path if merge report is enabled
if mergeReportPath != "" {
+1 -1
View File
@@ -105,7 +105,7 @@ func TestRunMerge_FromListPathWithSpaces(t *testing.T) {
defer restoreMergeState(saved)
spacedDir := filepath.Join(t.TempDir(), "my schema files")
if err := os.MkdirAll(spacedDir, 0755); err != nil {
if err := os.MkdirAll(spacedDir, 0o755); err != nil {
t.Fatal(err)
}
targetFile := filepath.Join(spacedDir, "target schema.json")
+15 -7
View File
@@ -1,6 +1,9 @@
package main
import (
"fmt"
"os"
"git.warky.dev/wdevs/relspecgo/pkg/readers"
"git.warky.dev/wdevs/relspecgo/pkg/writers"
)
@@ -10,17 +13,22 @@ func newReaderOptions(filePath, connString string) *readers.ReaderOptions {
FilePath: filePath,
ConnectionString: connString,
Prisma7: prisma7,
StrictDirectives: strictDirectives,
Progress: func(message string) {
fmt.Fprintf(os.Stderr, " → %s\n", message)
},
}
}
func newWriterOptions(outputPath, packageName string, flattenSchema bool, nullableTypes, nullableArrays string, continueOnError bool) *writers.WriterOptions {
return &writers.WriterOptions{
OutputPath: outputPath,
PackageName: packageName,
FlattenSchema: flattenSchema,
NullableTypes: nullableTypes,
NullableArrays: nullableArrays,
Prisma7: prisma7,
ContinueOnError: continueOnError,
OutputPath: outputPath,
PackageName: packageName,
FlattenSchema: flattenSchema,
NullableTypes: nullableTypes,
NullableArrays: nullableArrays,
Prisma7: prisma7,
ContinueOnError: continueOnError,
StrictDirectives: strictDirectives,
}
}
+12 -37
View File
@@ -2,50 +2,23 @@ package main
import (
"fmt"
"runtime/debug"
"time"
"github.com/spf13/cobra"
"git.warky.dev/wdevs/relspecgo/pkg/buildinfo"
)
// version/buildDate mirror pkg/buildinfo so existing call sites keep working.
// The actual values are set there via ldflags (see Makefile).
var (
// Version information, set via ldflags during build
version = "dev"
buildDate = "unknown"
prisma7 bool
noVersion bool
version = buildinfo.Version
buildDate = buildinfo.BuildDate
prisma7 bool
noVersion bool
silent bool
strictDirectives bool
)
func init() {
// If version wasn't set via ldflags, try to get it from build info
if version == "dev" {
if info, ok := debug.ReadBuildInfo(); ok {
// Try to get version from VCS
var vcsRevision, vcsTime string
for _, setting := range info.Settings {
switch setting.Key {
case "vcs.revision":
if len(setting.Value) >= 7 {
vcsRevision = setting.Value[:7]
}
case "vcs.time":
vcsTime = setting.Value
}
}
if vcsRevision != "" {
version = vcsRevision
}
if vcsTime != "" {
if t, err := time.Parse(time.RFC3339, vcsTime); err == nil {
buildDate = t.UTC().Format("2006-01-02 15:04:05 UTC")
}
}
}
}
}
var rootCmd = &cobra.Command{
Use: "relspec",
Short: "RelSpec - Database schema conversion and analysis tool",
@@ -72,6 +45,8 @@ func init() {
rootCmd.AddCommand(reportCmd)
rootCmd.PersistentFlags().BoolVar(&prisma7, "prisma7", false, "Use Prisma 7 generator conventions when reading/writing Prisma schemas")
rootCmd.PersistentFlags().BoolVar(&noVersion, "no-version", false, "Suppress the RelSpec version header")
rootCmd.PersistentFlags().BoolVar(&silent, "silent", false, "Suppress progress and status messages (errors are still shown)")
rootCmd.PersistentFlags().BoolVar(&strictDirectives, "strict-directives", false, "Fail on unknown or untranslatable DBML dialect directives (@postgres:, @sqlite:, …)")
}
// printVersionHeader prints the "RelSpec <version> (built: <date>)" banner
+73 -6
View File
@@ -24,6 +24,7 @@ var (
splitExcludeTables string
splitNullableTypes string
splitNullableArrays string
splitDryRun bool
)
var splitCmd = &cobra.Command{
@@ -115,6 +116,8 @@ func init() {
splitCmd.Flags().StringVar(&splitNullableTypes, "types", "", "Nullable type package for code-gen writers (bun/gorm): 'baselib' (default, Go pointer types), 'stdlib' (database/sql), or 'sqltypes'")
splitCmd.Flags().StringVar(&splitNullableArrays, "array-nullable", "", "Nullable PostgreSQL array representation for the Bun writer in stdlib/baselib --types mode: 'slice' (default, plain slice) or 'pointer_slice' (*[]T, distinguishes NULL from '{}')")
splitCmd.Flags().BoolVar(&splitDryRun, "dry-run", false, "Read, filter and validate the selection and print the plan without writing any output")
err := splitCmd.MarkFlagRequired("from")
if err != nil {
fmt.Fprintf(os.Stderr, "Error marking from flag as required: %v\n", err)
@@ -174,6 +177,26 @@ func runSplit(cmd *cobra.Command, args []string) error {
}
fmt.Fprintf(os.Stderr, " ✓ Filtered to: %d schema(s), %d table(s)\n\n", len(filteredDB.Schemas), filteredTables)
if splitDryRun {
if err := validateWriteTarget(filteredDB, splitTargetType, splitPackageName, "", ""); err != nil {
return fmt.Errorf("dry run validation failed: %w", err)
}
out := outWriter(cmd)
fmt.Fprintf(out, "RelSpec split plan (dry run - nothing written):\n")
fmt.Fprintf(out, " Input: %s database '%s'\n", splitSourceType, db.Name)
fmt.Fprintf(out, " Output: %s -> %s\n", splitTargetType, splitTargetPath)
fmt.Fprintf(out, " Selection: %s\n", splitSelection{
Schemas: parseCommaSeparated(splitSchemas),
Tables: parseCommaSeparated(splitTables),
ExcludeSchemas: parseCommaSeparated(splitExcludeSchema),
ExcludeTables: parseCommaSeparated(splitExcludeTables),
DatabaseName: splitDatabaseName,
}.summary())
printDryRunPlan(out, filteredDB)
fmt.Fprintf(os.Stderr, "=== Dry run complete: no output written ===\n\n")
return nil
}
// Write to target format
fmt.Fprintf(os.Stderr, "[3/3] Writing to target format...\n")
fmt.Fprintf(os.Stderr, " Format: %s\n", splitTargetType)
@@ -205,8 +228,52 @@ func runSplit(cmd *cobra.Command, args []string) error {
return nil
}
// filterDatabase filters the database based on provided criteria
// splitSelection is the schema/table selection for a split, independent of the
// CLI flag globals so the job runner can build one directly.
type splitSelection struct {
Schemas []string
Tables []string
ExcludeSchemas []string
ExcludeTables []string
DatabaseName string
}
// summary renders a one-line human description of the selection.
func (s splitSelection) summary() string {
var parts []string
if len(s.Schemas) > 0 {
parts = append(parts, "schemas="+strings.Join(s.Schemas, ","))
}
if len(s.Tables) > 0 {
parts = append(parts, "tables="+strings.Join(s.Tables, ","))
}
if len(s.ExcludeSchemas) > 0 {
parts = append(parts, "exclude_schemas="+strings.Join(s.ExcludeSchemas, ","))
}
if len(s.ExcludeTables) > 0 {
parts = append(parts, "exclude_tables="+strings.Join(s.ExcludeTables, ","))
}
if s.DatabaseName != "" {
parts = append(parts, "database_name="+s.DatabaseName)
}
if len(parts) == 0 {
return "(all schemas/tables)"
}
return strings.Join(parts, " ")
}
// filterDatabase filters the database based on the CLI split flags.
func filterDatabase(db *models.Database) (*models.Database, error) {
return filterDatabaseSelection(db, splitSelection{
Schemas: parseCommaSeparated(splitSchemas),
Tables: parseCommaSeparated(splitTables),
ExcludeSchemas: parseCommaSeparated(splitExcludeSchema),
ExcludeTables: parseCommaSeparated(splitExcludeTables),
})
}
// filterDatabaseSelection filters db down to the schemas/tables named by sel.
func filterDatabaseSelection(db *models.Database, sel splitSelection) (*models.Database, error) {
filteredDB := &models.Database{
Name: db.Name,
Description: db.Description,
@@ -220,11 +287,11 @@ func filterDatabase(db *models.Database) (*models.Database, error) {
Domains: db.Domains, // Keep domains for now
}
// Parse filter flags
includeSchemas := parseCommaSeparated(splitSchemas)
includeTables := parseCommaSeparated(splitTables)
excludeSchemas := parseCommaSeparated(splitExcludeSchema)
excludeTables := parseCommaSeparated(splitExcludeTables)
// Selection criteria
includeSchemas := sel.Schemas
includeTables := sel.Tables
excludeSchemas := sel.ExcludeSchemas
excludeTables := sel.ExcludeTables
// Convert table names to lowercase for case-insensitive matching
includeTablesLower := make(map[string]bool)
+1
View File
@@ -114,6 +114,7 @@ func runTempl(cmd *cobra.Command, args []string) error {
if err != nil {
return fmt.Errorf("failed to read source: %w", err)
}
finalizeCommentedRefs(db, stderrWarn)
// Print database stats
schemaCount := len(db.Schemas)
+2 -2
View File
@@ -10,7 +10,7 @@ import (
func writeTestTemplate(t *testing.T, path string) {
t.Helper()
content := []byte(`{{.Name}}`)
if err := os.WriteFile(path, content, 0644); err != nil {
if err := os.WriteFile(path, content, 0o644); err != nil {
t.Fatalf("failed to write template file %s: %v", path, err)
}
}
@@ -104,7 +104,7 @@ func TestRunTempl_FromListPathWithSpaces(t *testing.T) {
defer restoreTemplState(saved)
spacedDir := filepath.Join(t.TempDir(), "my schema files")
if err := os.MkdirAll(spacedDir, 0755); err != nil {
if err := os.MkdirAll(spacedDir, 0o755); err != nil {
t.Fatal(err)
}
file1 := filepath.Join(spacedDir, "users schema.json")
+2 -2
View File
@@ -66,7 +66,7 @@ func writeTestJSON(t *testing.T, path string, tableNames []string) {
if err != nil {
t.Fatalf("failed to marshal test JSON: %v", err)
}
if err := os.WriteFile(path, data, 0644); err != nil {
if err := os.WriteFile(path, data, 0o644); err != nil {
t.Fatalf("failed to write test file %s: %v", path, err)
}
}
@@ -100,7 +100,7 @@ func writeTestJSONWithSingleColumnType(t *testing.T, path, tableName, columnType
if err != nil {
t.Fatalf("failed to marshal test JSON: %v", err)
}
if err := os.WriteFile(path, data, 0644); err != nil {
if err := os.WriteFile(path, data, 0o644); err != nil {
t.Fatalf("failed to write test file %s: %v", path, err)
}
}
+5 -2
View File
@@ -4,13 +4,16 @@ import (
"fmt"
"github.com/spf13/cobra"
"git.warky.dev/wdevs/relspecgo/pkg/buildinfo"
)
var versionCmd = &cobra.Command{
Use: "version",
Short: "Print version information",
Run: func(cmd *cobra.Command, args []string) {
fmt.Printf("RelSpec %s\n", version)
fmt.Printf("Built: %s\n", buildDate)
fmt.Print(buildinfo.AsciiLogo)
fmt.Printf("RelSpec %s\n", buildinfo.Version)
fmt.Printf("Built: %s\n", buildinfo.BuildDate)
},
}
+152
View File
@@ -0,0 +1,152 @@
package main
import (
"context"
"fmt"
"io"
"io/fs"
"os"
"os/signal"
"path/filepath"
"strings"
"syscall"
"time"
)
// watchSnapshot maps a file path to its modification time and size.
type watchSnapshot map[string]string
// takeWatchSnapshot records the state of every file under the given paths.
// Directories are walked recursively. Anything at or below the excluded path
// (typically the output path) is skipped so regenerating output does not
// retrigger the watcher. Missing paths are simply absent from the snapshot, so
// creating them later counts as a change.
func takeWatchSnapshot(paths []string, exclude string) watchSnapshot {
snap := watchSnapshot{}
exclude = absPathOrSelf(exclude)
for _, root := range paths {
_ = filepath.WalkDir(root, func(p string, d fs.DirEntry, err error) error {
if err != nil {
return nil
}
if exclude != "" && isWithin(absPathOrSelf(p), exclude) {
if d.IsDir() {
return filepath.SkipDir
}
return nil
}
if d.IsDir() {
return nil
}
info, err := d.Info()
if err != nil {
return nil
}
snap[p] = fmt.Sprintf("%d-%d", info.ModTime().UnixNano(), info.Size())
return nil
})
}
return snap
}
func (s watchSnapshot) equal(o watchSnapshot) bool {
if len(s) != len(o) {
return false
}
for k, v := range s {
if o[k] != v {
return false
}
}
return true
}
func absPathOrSelf(p string) string {
if p == "" {
return ""
}
if abs, err := filepath.Abs(p); err == nil {
return abs
}
return p
}
// isWithin reports whether path equals dir or is located below it.
func isWithin(path, dir string) bool {
if path == dir {
return true
}
return strings.HasPrefix(path, dir+string(filepath.Separator))
}
// watchLoop runs fn once immediately and again whenever the watched paths
// change, until ctx is cancelled. Changes are debounced: fn runs only after
// the snapshot has stayed unchanged for one poll interval. Errors from fn are
// reported to w and do not stop the loop.
func watchLoop(ctx context.Context, w io.Writer, paths []string, exclude string, interval time.Duration, fn func() error) {
run := func() {
if err := fn(); err != nil {
fmt.Fprintf(w, "Error: %v\n", err)
}
fmt.Fprintf(w, "Watching for changes (Ctrl-C to stop)...\n")
}
last := takeWatchSnapshot(paths, exclude)
run()
ticker := time.NewTicker(interval)
defer ticker.Stop()
for {
select {
case <-ctx.Done():
return
case <-ticker.C:
}
cur := takeWatchSnapshot(paths, exclude)
if cur.equal(last) {
continue
}
// Debounce: wait until writes settle.
for settled := false; !settled; {
select {
case <-ctx.Done():
return
case <-time.After(interval):
}
next := takeWatchSnapshot(paths, exclude)
settled = next.equal(cur)
cur = next
}
fmt.Fprintf(w, "\nChange detected, regenerating...\n")
last = cur
run()
}
}
// runConvertWatch runs the conversion once and then again whenever the source
// files change, until interrupted.
func runConvertWatch(parent context.Context, w io.Writer, run func() error) error {
var paths []string
switch {
case len(convertFromList) > 0:
paths = convertFromList
case convertSourcePath != "":
paths = []string{convertSourcePath}
default:
return fmt.Errorf("--watch requires --from-path or --from-list (live database connections cannot be watched)")
}
if convertDryRun {
return fmt.Errorf("--watch cannot be combined with --dry-run")
}
if convertWatchInterval <= 0 {
return fmt.Errorf("--watch-interval must be positive")
}
if parent == nil {
parent = context.Background()
}
ctx, stop := signal.NotifyContext(parent, os.Interrupt, syscall.SIGTERM)
defer stop()
watchLoop(ctx, w, paths, convertTargetPath, convertWatchInterval, run)
return nil
}
+92
View File
@@ -0,0 +1,92 @@
package main
import (
"bytes"
"context"
"io"
"os"
"path/filepath"
"sync/atomic"
"testing"
"time"
)
func TestWatchSnapshotExcludesOutput(t *testing.T) {
dir := t.TempDir()
out := filepath.Join(dir, "out")
if err := os.MkdirAll(out, 0o755); err != nil {
t.Fatal(err)
}
src := filepath.Join(dir, "schema.dbml")
if err := os.WriteFile(src, []byte("a"), 0o644); err != nil {
t.Fatal(err)
}
before := takeWatchSnapshot([]string{dir}, out)
if err := os.WriteFile(filepath.Join(out, "gen.go"), []byte("x"), 0o644); err != nil {
t.Fatal(err)
}
if after := takeWatchSnapshot([]string{dir}, out); !before.equal(after) {
t.Errorf("writing into the excluded output path changed the snapshot")
}
if err := os.WriteFile(src, []byte("changed"), 0o644); err != nil {
t.Fatal(err)
}
if after := takeWatchSnapshot([]string{dir}, out); before.equal(after) {
t.Errorf("modifying a source file did not change the snapshot")
}
}
func TestWatchLoopRerunsOnChange(t *testing.T) {
dir := t.TempDir()
src := filepath.Join(dir, "schema.dbml")
if err := os.WriteFile(src, []byte("a"), 0o644); err != nil {
t.Fatal(err)
}
var runs atomic.Int32
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
done := make(chan struct{})
go func() {
defer close(done)
watchLoop(ctx, io.Discard, []string{src}, "", 10*time.Millisecond, func() error {
runs.Add(1)
return nil
})
}()
waitFor(t, func() bool { return runs.Load() == 1 })
if err := os.WriteFile(src, []byte("changed content"), 0o644); err != nil {
t.Fatal(err)
}
waitFor(t, func() bool { return runs.Load() == 2 })
cancel()
<-done
}
func TestRunConvertWatchValidation(t *testing.T) {
oldPath, oldList, oldDry, oldInt := convertSourcePath, convertFromList, convertDryRun, convertWatchInterval
defer func() {
convertSourcePath, convertFromList, convertDryRun, convertWatchInterval = oldPath, oldList, oldDry, oldInt
}()
convertSourcePath, convertFromList, convertDryRun, convertWatchInterval = "", nil, false, time.Second
if err := runConvertWatch(context.Background(), &bytes.Buffer{}, nil); err == nil {
t.Error("expected error without --from-path/--from-list")
}
convertSourcePath, convertDryRun = "x.dbml", true
if err := runConvertWatch(context.Background(), &bytes.Buffer{}, nil); err == nil {
t.Error("expected error combining --watch with --dry-run")
}
}
func waitFor(t *testing.T, cond func() bool) {
t.Helper()
deadline := time.Now().Add(5 * time.Second)
for time.Now().Before(deadline) {
if cond() {
return
}
time.Sleep(5 * time.Millisecond)
}
t.Fatal("condition not met in time")
}
+115
View File
@@ -0,0 +1,115 @@
# DBML Dialect Directives
DBML has no dialect-neutral way to express database-specific features such as
PostgreSQL table partitioning or SQLite `WITHOUT ROWID`. RelSpec adds **dialect
directives** — explicit, parseable lines embedded in a `.dbml` file that are:
- stored losslessly in the intermediate model (under each object's `Metadata`),
- preserved unchanged through a `DBML → model → DBML` round-trip,
- translated to SQL **only** by the writer for the matching dialect
(`@postgres:` clauses appear in PostgreSQL output, never in SQLite output, and
vice-versa).
## Grammar
A directive is a single line, matched on its trimmed content:
```
@<namespace>[(<target>)]: <args>
```
| Part | Rules |
|------|-------|
| `namespace` | `^[a-z][a-z0-9_]*$` — e.g. `postgres`, `sqlite`. Future dialects allowed. |
| `(target)` | Optional. A **column name** only, valid only on a directive line inside a table body. Bare or single/double quoted. |
| `args` | Everything after the first `:`, trimmed. Otherwise preserved **verbatim**. Must be non-empty. |
The **key** of a directive is derived: the lowercased first whitespace-delimited
token of `args` (`partition by RANGE (created_at)` → `partition`). It drives
duplicate detection and writer dispatch.
## Location
Where the line appears determines which object it attaches to:
| Position in the file | Attaches to |
|----------------------|-------------|
| Before the first `Table {` | database (`db.Metadata`) |
| Table body, no `(target)` | that table |
| Table body, `(col)` target | column `col` of that table (error if `col` is unknown) |
| Inside an `indexes { }` block | the **most recently listed** index entry in that block; `(target)` is not allowed |
```dbml
@postgres: search_path myapp -- database
Table myapp.events {
id bigint [pk]
created_at timestamp [not null]
@postgres(id): identity always -- column "id"
@postgres: partition by RANGE (created_at) -- table
@postgres: tablespace fast_data -- table
@sqlite: without rowid -- table
indexes {
(created_at) [name: 'idx_events_created']
@postgres: with (fillfactor=90) -- index "idx_events_created"
@postgres: tablespace idx_space -- index "idx_events_created"
}
}
```
## Duplicate policy
- **Repeatable by default** — every directive with the same `(namespace, key)` at
one location is kept, in source order.
- **Singletons** raise a line-numbered error on a second occurrence at the same
location. Current singletons: `postgres` `partition`, `tablespace`, `inherits`,
`storage`, `compression`, `identity`; `sqlite` `without`, `strict`, `collate`.
## Strict mode
CLI flag `--strict-directives` (also `ReaderOptions.StrictDirectives` /
`WriterOptions.StrictDirectives`):
- **Reader**: an unknown namespace or key is a hard error. Without strict mode it
is stored and preserved silently, and round-trips unchanged.
- **PostgreSQL / SQLite writer**: a directive for **that** writer's own dialect
whose key it cannot translate is a hard error. Without strict mode, translatable
keys are emitted and the rest are skipped. Directives for other dialects are
always ignored, never emitted.
## Errors
All are line-numbered (`dbml: line N: …`):
- no colon, or empty `args`
- namespace empty or not matching `[a-z][a-z0-9_]*`
- `(target)` naming an unknown column, or used at the top level / in an `indexes` block
- a directive in the catalog used at a location it is not valid for
- duplicate singleton at the same location
- (strict mode) unknown `(namespace, key)`
## Supported directive matrix
### `@postgres`
| Key | Locations | SQL emitted | Notes |
|-----|-----------|-------------|-------|
| `partition` | table | `PARTITION BY <args>` appended to `CREATE TABLE` | e.g. `@postgres: partition by RANGE (created_at)` |
| `inherits` | table | `INHERITS (<args>)` — args verbatim | |
| `with` | table, index | `WITH (<params>)` | On an index, wins over `WITH` derived from the index comment. `@postgres: with (fillfactor=90)` |
| `tablespace` | table, index | `TABLESPACE <name>` | Emitted after `WITH`, before `WHERE` on indexes |
| `storage` | column | `STORAGE <mode>` in the column definition | e.g. `@postgres(blob): storage external` |
| `compression` | column | `COMPRESSION <method>` | |
| `identity` | column | `identity always` → `GENERATED ALWAYS AS IDENTITY`; `identity default` / `identity by default` → `GENERATED BY DEFAULT AS IDENTITY` | |
### `@sqlite`
| Key | Locations | SQL emitted | Notes |
|-----|-----------|-------------|-------|
| `without` | table | `WITHOUT ROWID` table option | `@sqlite: without rowid` |
| `strict` | table | `STRICT` table option | `WITHOUT ROWID` is emitted before `STRICT` |
| `collate` | column | ` COLLATE <name>` in the column definition | e.g. `@sqlite(name): collate NOCASE` |
Unknown namespaces and keys not in these tables are still preserved losslessly
(and round-trip through the DBML writer) whenever strict mode is off.
+103
View File
@@ -0,0 +1,103 @@
# Format Usage Examples
Examples for `relspec convert` covering the file-based reader and writer
formats. The "Writers" and "Readers" sections below were run against
`examples/test_schema.dbml`. The cross-format and live-database examples were
not run; they follow the flags shown in `relspec convert --help` and require
matching input files or reachable databases.
Any reader can be combined with any writer: pick `--from`/`--from-path` for the
source and `--to`/`--to-path` for the target. Add `--silent` to suppress progress
output.
## Writers: DBML to every format
```bash
S="--from dbml --from-path examples/test_schema.dbml"
relspec convert $S --to json --to-path schema.json
relspec convert $S --to yaml --to-path schema.yaml
relspec convert $S --to dctx --to-path schema.dctx
relspec convert $S --to drawdb --to-path schema.drawdb.json
relspec convert $S --to graphql --to-path schema.graphql
relspec convert $S --to prisma --to-path schema.prisma
relspec convert $S --to pgsql --to-path schema.pg.sql
relspec convert $S --to mssql --to-path schema.mssql.sql
relspec convert $S --to sqlite --to-path schema.sqlite.sql
relspec convert $S --to drizzle --to-path schema.ts
relspec convert $S --to typeorm --to-path entities.ts
relspec convert $S --to gorm --to-path models.go --package models
relspec convert $S --to bun --to-path models.go --package models
```
Notes:
- Code-generation writers (`gorm`, `bun`) take `--package`. They also accept
`--types baselib|stdlib|sqltypes` to choose the nullable type package.
- When `--to-path` is a directory it must already exist.
- `sqlite` output automatically flattens `schema.table` names. Use
`--flatten-schema` for other formats if the target has no schema support.
- `dctx` supports a single schema only; use `--schema <name>` to select one.
## Readers: file-based formats into DBML (or JSON where noted)
```bash
relspec convert --from json --from-path schema.json --to dbml --to-path out.dbml
relspec convert --from yaml --from-path schema.yaml --to dbml --to-path out.dbml
relspec convert --from dctx --from-path schema.dctx --to dbml --to-path out.dbml
relspec convert --from drawdb --from-path schema.drawdb.json --to dbml --to-path out.dbml
relspec convert --from graphql --from-path schema.graphql --to dbml --to-path out.dbml
relspec convert --from prisma --from-path schema.prisma --to dbml --to-path out.dbml
relspec convert --from drizzle --from-path schema.ts --to dbml --to-path out.dbml
relspec convert --from typeorm --from-path entities.ts --to dbml --to-path out.dbml
relspec convert --from bun --from-path models.go --to dbml --to-path out.dbml
relspec convert --from gorm --from-path models.go --to json --to-path out.json
```
Code-first readers (`gorm`, `bun`, `drizzle`, `typeorm`) accept a single file or a
directory of model files.
> Known issue: reading GORM models and writing DBML currently panics in the DBML
> writer (`pkg/writers/dbml/writer.go`, `constraintToDBML`). Use another target
> such as JSON until this is fixed.
## Cross-format combinations
```bash
# ORM models to SQL DDL
relspec convert --from gorm --from-path models.go --to pgsql --to-path schema.sql
# Prisma to Drizzle
relspec convert --from prisma --from-path schema.prisma --to drizzle --to-path schema.ts
# DrawDB diagram to GraphQL
relspec convert --from drawdb --from-path diagram.json --to graphql --to-path schema.graphql
# Merge several files while converting
relspec convert --from json --from-list "a.json,b.json" --to yaml --to-path merged.yaml
```
## Live databases
These need a reachable database:
```bash
# PostgreSQL
relspec convert --from pgsql --from-conn "postgres://user:pass@localhost:5432/mydb" \
--to dbml --to-path schema.dbml
# SQL Server
relspec convert --from mssql --from-conn "<mssql connection string>" \
--to json --to-path schema.json
# SQLite database file (--from-conn takes the file path)
relspec convert --from sqlite --from-conn ./app.db --to dbml --to-path schema.dbml
```
## Formats outside `convert`
- `sqldir` (SQL script directory reader) and `sqlexec` (SQL execution writer) are
used by `relspec scripts` and `relspec job`, and `sqldir` by `relspec diff`.
See [SCRIPTS_COMMAND.md](SCRIPTS_COMMAND.md) and [JOB_FILES.md](JOB_FILES.md).
- The `template` writer is exposed through `relspec templ`. See
[TEMPLATE_MODE.md](TEMPLATE_MODE.md).
+181 -28
View File
@@ -9,10 +9,11 @@ relspec job run build-schema --plan # validate + print plan, execute nothing
relspec job run build-schema # run the job (and its dependencies)
```
## Design contract (first release)
## Design contract
This is the smallest coherent contract that is safe and useful end to end.
Anything not listed under "Supported" is intentionally deferred.
This is a deliberately small, safe contract. Every capability is offline-testable
except live database execution (`scripts-exec`), which is validated and planned
offline and only connects at run time.
### Not a shell
@@ -24,13 +25,15 @@ means adding a vetted adapter in the RelSpec source.
|----------------|--------------------------------------------------------------------|
| `convert` | read one or more input schemas, additively merge them, write one output |
| `merge` | like `convert` but requires ≥2 inputs and exposes `skip_*` merge options |
| `split` | read one or more schemas, keep the selected schemas/tables, write one output |
| `scripts-list` | deterministically list SQL scripts across one or more directories |
| `scripts-exec` | execute SQL scripts across one or more directories against a live PostgreSQL database |
| `templ` | apply a custom Go text template to one or more input schemas |
| `inspect` | validate one or more schemas against rules and write a report |
| `diff` | compare exactly two schemas and write a differences report |
Deferred (documented, not implemented here): `scripts` execution against a live
database, `split`, `inspect`, `diff`, `templ`, job-to-job output wiring,
log rotation/retention. Live SQL execution already exists as
`relspec scripts execute`; wiring it into the job runner is a follow-up because
it needs live database credentials and cannot be covered by offline tests.
`convert`, `merge` and `split` are **producers**: their file output can be fed
directly into another job with `from_job` (see below).
### Discovery and precedence
@@ -48,12 +51,20 @@ already forbid duplicate keys within a single file.
### Paths
* Every path (`inputs[].path`, `output.path`, `script_dirs[]`, `logfile`) is
**relative to the directory containing the job file that declared the job**,
not the process working directory.
* Every path (`inputs[].path`, `output.path`, `report.path`, `rules`,
`script_dirs[]`, `template`, `logfile`) is **relative to the directory
containing the job file that declared the job**, not the process working
directory.
* Those path fields support `${NAME}` environment-variable references. They
are expanded immediately before execution; a missing variable is an error.
The expanded value is still subject to all relative-path and symlink safety
checks.
* Absolute paths, `~`-relative paths and any path that resolves outside the job
file directory (`../`, `a/../../b`, …) are **rejected during validation** —
before anything runs.
* At run time each path is additionally resolved through its symlinks: a symlink
inside the job-file directory that points outside it is rejected before the
path is opened.
### Credentials
@@ -75,72 +86,140 @@ already forbid duplicate keys within a single file.
first. Nothing is read, written, connected to, or executed if validation fails.
Checks include:
* schema `version` (must be `1`), unknown YAML fields rejected
* schema `version` — **forward-permissive**: any version `>= 1` is accepted.
An omitted `version` is treated as the current one. A version newer than this
build understands loads best-effort (unknown YAML fields are ignored and a
warning is printed); at the current version unknown YAML fields are still
rejected.
* duplicate job names across files
* unknown / missing `command`
* per-command input/output shape (`convert`/`merge` need inputs + output;
`scripts-list` needs `script_dirs` and forbids inputs/output)
* per-command input/output shape:
* `convert` needs ≥1 input + output; `merge` needs ≥2 inputs + output
* `split` needs ≥1 input + a file output, plus an optional `select:` block
* `scripts-list` needs `script_dirs` and forbids inputs/output
* `scripts-exec` needs `script_dirs` and `output.conn_env` (pgsql only)
* `inspect` needs ≥1 input + `report:` (format `markdown`|`json`)
* `diff` needs **exactly 2** inputs + `report:` (format `summary`|`json`|`html`)
* unknown input/output `format`
* `from_job` targets exist, are producers (`convert`/`merge`/`split`) and write a
single-file output
* path traversal / absolute / home-relative paths
* `depends_on` targets exist
* dependency cycles (reported as `a -> b -> c -> a`)
* `depends_on` and `from_job` targets exist
* dependency cycles over the combined `depends_on` + `from_job` graph
(reported as `a -> b -> c -> a`)
Then, immediately before running, per-job pre-flight resolves paths and checks:
* every input file exists and is a file
* every input file exists and is a file (a `from_job` input is exempt — its
producer runs earlier in the same plan)
* every `script_dir` exists and is a directory
* every `conn_env` variable is set
* `output.path` does not already exist unless `output.overwrite: true`
* `output.path` / `report.path` does not already exist unless the matching
`overwrite: true` is set
* `rules` (inspect), when given, exists and is a file
* symlinks in every resolved path stay inside the job-file directory
If any pre-flight check fails for **any** job in the plan, **no** job runs.
### Execution and exit codes
* `relspec job run <name>` runs the job's `depends_on` closure first, in
topological order (deterministic), then the job. `--no-deps` runs only the
named job.
* `relspec job run <name>` runs the job's dependency closure first
(`depends_on` plus any `from_job` producers), in topological order
(deterministic), then the job. `--no-deps` runs only the named job and is
incompatible with `from_job` inputs.
* `--dry-run` (alias `--plan`) prints the resolved plan and exits 0 without
touching inputs, outputs or databases.
* A failing job returns the underlying non-zero status (the process exits 1)
and the error names the job. The logfile records `FAILED: <error>`; a
successful job records `OK`. No separate success-marker file is written, so a
failure can never leave a stale "success".
* `inspect` fails the job when the report contains rule **errors** (enforced
rules); warnings do not fail it. `diff` never fails on differences.
* Single-file outputs and reports are written to a temporary file in the target
directory and atomically renamed into place, so an interrupted run never
leaves a partial file. Directory-emitting formats (`gorm`, `bun`, `drizzle`,
`typeorm`, `prisma`) are written in place.
### Logfile rotation
When a job has a `logfile`, it is size-rotated before each run. Defaults are
**5 MB** with **3** rotated files kept (`build.log` → `build.log.1` → …). Override
per job with `log_max_size` / `log_keep`, or for a whole file with a top-level
`defaults:` block. `log_max_size` accepts `B`/`KB`/`MB`/`GB` suffixes (e.g.
`"512KB"`, `"5MB"`).
## Schema reference
```yaml
version: 1 # required, must be 1
version: 1 # optional; any value >= 1 is accepted
defaults: # optional, file-wide
log_max_size: 5MB # B / KB / MB / GB
log_keep: 3
jobs:
<job-name>:
command: convert | merge | scripts-list # required
command: convert | merge | split | scripts-list | scripts-exec | templ | inspect | diff
description: "free text" # optional, shown by `job list`
depends_on: [other-job, ...] # optional
inputs: # convert (≥1) / merge (≥2)
inputs: # convert (≥1) / merge (≥2) / split (≥1) / inspect (≥1) / diff (exactly 2)
- path: relative/file.dbml # file inputs
format: dbml
- format: pgsql # live-connection inputs
conn_env: SOURCE_DB_URL # env var NAME
script_dirs: # scripts-list (≥1)
- from_job: build-schema # consume another job's file output
script_dirs: # scripts-list / scripts-exec (≥1)
- migrations/core
- migrations/tenant
output: # convert / merge (required)
template: templates/schema.tmpl # templ (required)
mode: table # templ: database/schema/script/table
filename_pattern: "{{.Name}}.go" # templ multi-output modes
select: # split (optional; default = keep everything)
schemas: [public]
tables: [users, orders]
exclude_schemas: []
exclude_tables: []
database_name: SubsetDB # optional rename of the output database
rules: .relspec-rules.yaml # inspect (optional; built-in defaults if omitted)
report: # inspect (required) / diff (required)
format: json # inspect: markdown|json ; diff: summary|json|html
path: build/report.json # required, except a diff "summary" (goes to the log)
overwrite: false
output: # convert / merge / split (required); scripts-exec (required, conn_env)
format: pgsql
path: build/schema.sql # file output, OR:
conn_env: TARGET_DB_URL # execute against DB (pgsql only)
# default: reads live DB, executes only the diff
# (file output = full DDL); set options.full_ddl to force full DDL
overwrite: false # default false
options:
flatten_schema: false
schema: public
package: models # for gorm/bun output
continue_on_error: false # pgsql output
types: sqltypes # Bun/GORM nullable types: baselib|stdlib|sqltypes
array_nullable: pointer_slice # Bun: slice|pointer_slice
continue_on_error: false # pgsql / scripts-exec output
full_ddl: false # pgsql database output: true = run full DDL; default diffs against live DB
skip_relations: false # merge only
skip_enums: false
skip_views: false
skip_domains: false
skip_sequences: false
logfile: .relspec/log/<job-name>.log # optional; appended to
logfile: .relspec/log/<job-name>.log # optional; appended to, size-rotated
log_max_size: 5MB # optional per-job override
log_keep: 3 # optional per-job override
```
For `templ`, `inputs` use the same file or `pgsql`/`conn_env` source forms as
schema conversion. `output` is optional (empty means stdout); when present it
contains `path` and `overwrite`, plus the explicit `format: text` marker. The
marker is templ-only: it describes arbitrary text produced by a Go template,
not a database schema format. It is optional for compatibility with older
manifests and has no effect on template execution.
A `from_job` input takes no `path`, `format` or `conn_env`: it resolves to the
named job's `output.path` and inherits its format, and implies a dependency on
that job. The producer must be a `convert`, `merge` or `split` job writing a
single-file output.
### Supported input formats
`dbml`, `dctx`, `drawdb`, `graphql`, `json`, `yaml`, `gorm`, `bun`, `drizzle`,
@@ -221,3 +300,77 @@ jobs:
path: snapshots/prod.dbml
overwrite: true
```
### Chain jobs with `from_job`, then lint the result
```yaml
version: 1
jobs:
build-json:
command: convert
inputs:
- { path: schema/core.dbml, format: dbml }
- { path: schema/tenant.dbml, format: dbml }
output: { format: json, path: build/schema.json, overwrite: true }
lint-schema:
command: inspect
inputs:
- from_job: build-json # implies depends_on: [build-json]
rules: .relspec-rules.yaml # optional; built-in rules if omitted
report:
format: markdown
path: build/lint-report.md
overwrite: true
```
`relspec job run lint-schema` runs `build-json` first, then inspects its output.
The job fails (exit 1) if any enforced rule is violated.
### Split a subset out of a larger schema
```yaml
version: 1
jobs:
posts-only:
command: split
inputs:
- { path: schema/core.dbml, format: dbml }
- { path: schema/tenant.dbml, format: dbml }
select:
tables: [posts]
output: { format: dbml, path: build/posts.dbml, overwrite: true }
```
### Diff two schemas
```yaml
version: 1
jobs:
drift:
command: diff
inputs: # exactly two
- { path: build/schema.json, format: json }
- format: pgsql
conn_env: PROD_DB_URL
report:
format: summary # summary → logfile; json/html need a path
```
`diff` reports differences and always exits 0.
### Execute migration scripts against a live database
```yaml
version: 1
jobs:
apply-migrations:
command: scripts-exec
script_dirs:
- migrations/core
- migrations/tenant
output:
conn_env: TARGET_DB_URL # pgsql only; no path
options:
continue_on_error: false
logfile: .relspec/log/apply-migrations.log
```
+80
View File
@@ -0,0 +1,80 @@
# RelSpec job examples
Run these commands from this directory:
```bash
cd examples/jobs
# List every job from relspec.yml and the additional relspec.*.yml files.
relspec job list
# Validate a job and print its resolved plan without reading or writing data.
relspec job run build-schema --plan
# Build PostgreSQL DDL from the two DBML inputs.
relspec job run build-schema
# Run a job's dependency chain. This runs build-schema first, then build-json.
relspec job run build-json
# Inspect build-json's output. The from_job input adds the dependency automatically.
relspec job run lint-schema
# Extract only the posts table.
relspec job run posts-only
# List migration scripts in deterministic order without connecting to a database.
relspec job run migration-order
# Render one TypeScript file per table using the example template.
relspec job run generate-typescript
# Compare the two example schemas and write the summary to the logfile.
relspec job run compare-schemas
# Use an environment variable in an output path.
export VST=build/typescript
relspec job run generate-typescript --plan
relspec job run generate-typescript
```
Database jobs use environment variables for connection strings. Set them in
the shell before running the jobs; do not put connection strings in YAML:
```bash
export SOURCE_DB_URL='postgres://user:password@localhost/source_db'
export TARGET_DB_URL='postgres://user:password@localhost/target_db'
# Read the source database and write a local DBML snapshot.
relspec job run snapshot-database
# Preview migration execution without connecting to PostgreSQL.
relspec job run apply-migrations --plan
# Execute the migrations against TARGET_DB_URL.
relspec job run apply-migrations
```
The `apply-migrations` job demonstrates execution from multiple directories:
```yaml
script_dirs:
- migrations/core
- migrations/tenant
output:
format: pgsql
conn_env: TARGET_DB_URL
```
RelSpec combines the SQL files from both folders in deterministic order before
executing them against the configured PostgreSQL database.
To use a different directory or manifest, pass `--dir` or `--file`:
```bash
relspec job list --dir /path/to/project
relspec job run build-schema --file /path/to/project/relspec.yml --plan
```
See [../../docs/JOB_FILES.md](../../docs/JOB_FILES.md) for the complete job
file reference.
+29
View File
@@ -0,0 +1,29 @@
# Database-oriented examples. Set the referenced environment variables before
# running these jobs; connection strings never belong in a job file.
version: 1
jobs:
snapshot-database:
command: convert
description: Read a PostgreSQL database and write a DBML snapshot
inputs:
- format: pgsql
conn_env: SOURCE_DB_URL
output:
format: dbml
path: build/database-snapshot.dbml
overwrite: true
apply-migrations:
command: scripts-exec
description: Execute migrations from core and tenant folders against PostgreSQL
# All SQL files from these directories are discovered in deterministic order.
script_dirs:
- migrations/core
- migrations/tenant
output:
format: pgsql
conn_env: TARGET_DB_URL
options:
continue_on_error: false
logfile: .relspec/log/apply-migrations.log
+32
View File
@@ -0,0 +1,32 @@
# Reporting and templating examples. These jobs are discovered together with
# relspec.yml when running `relspec job list` from this directory.
version: 1
jobs:
generate-typescript:
command: templ
description: Render one TypeScript file per table from the merged schema
inputs:
- path: schema/core.dbml
format: dbml
- path: schema/tenant.dbml
format: dbml
template: templates/schema.tmpl
mode: table
filename_pattern: "{{.Name}}.ts"
output:
format: text
path: "${VST}"
overwrite: true
compare-schemas:
command: diff
description: Compare the two example schemas and print a summary
inputs:
- path: schema/core.dbml
format: dbml
- path: schema/tenant.dbml
format: dbml
report:
format: summary
logfile: .relspec/log/compare-schemas.log
+34
View File
@@ -4,7 +4,14 @@
# relspec job list
# relspec job run build-schema --plan
# relspec job run build-schema
# relspec job run lint-schema # inspect, consuming build-json's output
version: 1
# File-wide defaults. Individual jobs may override log_max_size / log_keep.
defaults:
log_max_size: 2MB
log_keep: 5
jobs:
build-schema:
command: convert
@@ -43,3 +50,30 @@ jobs:
- migrations/core
- migrations/tenant
logfile: .relspec/log/migration-order.log
lint-schema:
command: inspect
description: Validate build-json's output against the built-in rules
# No depends_on needed: the from_job input implies a dependency on build-json.
inputs:
- from_job: build-json
report:
format: markdown
path: build/lint-report.md
overwrite: true
logfile: .relspec/log/lint-schema.log
posts-only:
command: split
description: Extract just the posts table into its own DBML file
inputs:
- path: schema/core.dbml
format: dbml
- path: schema/tenant.dbml
format: dbml
select:
tables: [posts]
output:
format: dbml
path: build/posts.dbml
overwrite: true
+6
View File
@@ -0,0 +1,6 @@
// Code generated by RelSpec. DO NOT EDIT.
export interface {{.Name}} {
{{- range values .Table.Columns}}
{{.Name}}: {{.Type}};
{{- end}}
}
+7 -7
View File
@@ -1,6 +1,6 @@
module git.warky.dev/wdevs/relspecgo
go 1.25.7
go 1.25.13
require (
github.com/gdamore/tcell/v2 v2.13.9
@@ -12,7 +12,7 @@ require (
github.com/stretchr/testify v1.11.1
github.com/uptrace/bun v1.2.18
github.com/uptrace/bun/dialect/pgdialect v1.2.18
golang.org/x/text v0.37.0
golang.org/x/text v0.39.0
gopkg.in/yaml.v3 v3.0.1
modernc.org/sqlite v1.50.1
)
@@ -42,11 +42,11 @@ require (
github.com/tmthrgd/go-hex v0.0.0-20190904060850-447a3041c3bc // indirect
github.com/vmihailenco/msgpack/v5 v5.4.1 // indirect
github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect
golang.org/x/crypto v0.51.0 // indirect
golang.org/x/sync v0.20.0 // indirect
golang.org/x/sys v0.44.0 // indirect
golang.org/x/term v0.43.0 // indirect
golang.org/x/tools v0.45.0 // indirect
golang.org/x/crypto v0.53.0 // indirect
golang.org/x/net v0.56.0 // indirect
golang.org/x/sync v0.21.0 // indirect
golang.org/x/sys v0.46.0 // indirect
golang.org/x/term v0.44.0 // indirect
modernc.org/libc v1.72.3 // indirect
modernc.org/mathutil v1.7.1 // indirect
modernc.org/memory v1.11.0 // indirect
+16 -16
View File
@@ -102,49 +102,49 @@ github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5t
go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
golang.org/x/crypto v0.51.0 h1:IBPXwPfKxY7cWQZ38ZCIRPI50YLeevDLlLnyC5wRGTI=
golang.org/x/crypto v0.51.0/go.mod h1:8AdwkbraGNABw2kOX6YFPs3WM22XqI4EXEd8g+x7Oc8=
golang.org/x/crypto v0.53.0 h1:QZ4Muo8THX6CizN2vPPd5fBGHyogrdK9fG4wLPFUsto=
golang.org/x/crypto v0.53.0/go.mod h1:DNLU434OwVakk9PzuwV8w62mAJpRJL3vsgcfp4Qnsio=
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
golang.org/x/mod v0.36.0 h1:JJjpVx6myfUsUdAzZuOSTTmRE0PfZeNWzzvKrP7amb4=
golang.org/x/mod v0.36.0/go.mod h1:moc6ELqsWcOw5Ef3xVprK5ul/MvtVvkIXLziUOICjUQ=
golang.org/x/mod v0.37.0 h1:vF1DjpVEshcIqoEaauuHebaLk1O1forxjxBaVn884JQ=
golang.org/x/mod v0.37.0/go.mod h1:m8S8VeM9r4dzDwjrKO0a1sZP3YjeMamRRlD+fmR2Q/0=
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
golang.org/x/net v0.54.0 h1:2zJIZAxAHV/OHCDTCOHAYehQzLfSXuf/5SoL/Dv6w/w=
golang.org/x/net v0.54.0/go.mod h1:Sj4oj8jK6XmHpBZU/zWHw3BV3abl4Kvi+Ut7cQcY+cQ=
golang.org/x/net v0.56.0 h1:Rw8j/hFzGvJUZwNBXnAtf5sVDVt+65SK2C7IxCxZt5o=
golang.org/x/net v0.56.0/go.mod h1:D3Ku6r+V6JROoZK144D2XfMHFcMq/0zSfLelVTCFKec=
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4=
golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
golang.org/x/sync v0.21.0 h1:HLII4xRRTtCRkxYp4HNFF0Js/Og6q2i++KXbg0gHCwM=
golang.org/x/sync v0.21.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.44.0 h1:ildZl3J4uzeKP07r2F++Op7E9B29JRUy+a27EibtBTQ=
golang.org/x/sys v0.44.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
golang.org/x/sys v0.46.0 h1:noSf2Fq6F8DBgS+LysIkx7rIExoNHJsxOAtPp4rthXw=
golang.org/x/sys v0.46.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
golang.org/x/term v0.43.0 h1:S4RLU2sB31O/NCl+zFN9Aru9A/Cq2aqKpTZJ6B+DwT4=
golang.org/x/term v0.43.0/go.mod h1:lrhlHNdQJHO+1qVYiHfFKVuVioJIheAc3fBSMFYEIsk=
golang.org/x/term v0.44.0 h1:0rLvDRCtNj0gZkyIXhCyOb2OAzEhLVqc4B+hrsBhrmc=
golang.org/x/term v0.44.0/go.mod h1:7ze4MdzUzLXpSAoFP1H0bOI9aXDqveSvatT5vKcFh2Y=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
golang.org/x/text v0.37.0 h1:Cqjiwd9eSg8e0QAkyCaQTNHFIIzWtidPahFWR83rTrc=
golang.org/x/text v0.37.0/go.mod h1:a5sjxXGs9hsn/AJVwuElvCAo9v8QYLzvavO5z2PiM38=
golang.org/x/text v0.39.0 h1:UbZz4pLOvn600D6Oh6GGEI6VAmndrEBLv8/6BEXzyus=
golang.org/x/text v0.39.0/go.mod h1:3UwRclnC2g0TU9x8PZiyfOajCd1zaUNHF9cvqcQZ+ZM=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
golang.org/x/tools v0.45.0 h1:18qN3FAooORvApf5XjCXgsuayZOEtXf6JK18I3+ONa8=
golang.org/x/tools v0.45.0/go.mod h1:LuUGqqaXcXMEFEruIVJVm5mgDD8vww/z/SR1gQ4uE/0=
golang.org/x/tools v0.47.0 h1:7Kn5x/d1svx/PzryTsqeoZN4TZwqeH5pGWjefhLi/1Q=
golang.org/x/tools v0.47.0/go.mod h1:dFHnyTvFWY212G+h7ZY4Vsp/K3U4/7W9TyVaAul8uCA=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
+2 -2
View File
@@ -1,6 +1,6 @@
# Maintainer: Hein (Warky Devs) <hein@warky.dev>
pkgname=relspec
pkgver=1.0.74
pkgver=1.0.85
pkgrel=1
pkgdesc="RelSpec is a comprehensive database relations management tool that reads, transforms, and writes database table specifications across multiple formats and ORMs."
arch=('x86_64' 'aarch64')
@@ -15,7 +15,7 @@ build() {
export CGO_ENABLED=0
go build \
-trimpath \
-ldflags "-X git.warky.dev/wdevs/relspecgo/cmd/relspec.version=$pkgver" \
-ldflags "-X 'git.warky.dev/wdevs/relspecgo/pkg/buildinfo.Version=v$pkgver' -X 'git.warky.dev/wdevs/relspecgo/pkg/buildinfo.BuildDate=$(date -u +"%Y-%m-%d %H:%M:%S UTC")'" \
-o "$pkgname" ./cmd/relspec
}
+2 -2
View File
@@ -1,5 +1,5 @@
Name: relspec
Version: 1.0.74
Version: 1.0.85
Release: 1%{?dist}
Summary: RelSpec is a comprehensive database relations management tool that reads, transforms, and writes database table specifications across multiple formats and ORMs.
@@ -25,7 +25,7 @@ DBML, GraphQL, and more.
export CGO_ENABLED=0
go build \
-trimpath \
-ldflags "-X git.warky.dev/wdevs/relspecgo/cmd/relspec.version=%{version}" \
-ldflags "-X 'git.warky.dev/wdevs/relspecgo/pkg/buildinfo.Version=v%{version}' -X 'git.warky.dev/wdevs/relspecgo/pkg/buildinfo.BuildDate=$(date -u +"%%Y-%%m-%%d %%H:%%M:%%S UTC")'" \
-o %{name} ./cmd/relspec
%install
+4 -1
View File
@@ -137,7 +137,10 @@ func TestScanDir_OrdersByPriorityThenSequence(t *testing.T) {
t.Fatalf("expected 4 items, got %d", len(items))
}
type ps struct{ p int; s uint }
type ps struct {
p int
s uint
}
want := []ps{{1, 1}, {1, 2}, {2, 1}, {2, 2}}
for i, w := range want {
got := ps{items[i].Priority, items[i].Sequence}
+65
View File
@@ -0,0 +1,65 @@
// Package buildinfo exposes the RelSpec version and build date so that both the
// CLI and the schema writers can stamp generated output with the same values.
package buildinfo
import (
"fmt"
"runtime/debug"
"time"
)
// Version and BuildDate are set via -ldflags at build time (see Makefile). When
// built without ldflags they are backfilled from the Go module build info.
var (
Version = "dev"
BuildDate = "unknown"
)
func init() {
if Version != "dev" {
return
}
info, ok := debug.ReadBuildInfo()
if !ok {
return
}
var rev, vcsTime string
for _, s := range info.Settings {
switch s.Key {
case "vcs.revision":
if len(s.Value) >= 7 {
rev = s.Value[:7]
}
case "vcs.time":
vcsTime = s.Value
}
}
if rev != "" {
Version = rev
}
if t, err := time.Parse(time.RFC3339, vcsTime); err == nil {
BuildDate = t.UTC().Format("2006-01-02 15:04:05 UTC")
}
}
// GeneratedComment returns the one-line provenance string embedded in generated
// files, e.g. "RelSpec dev (built: unknown)".
func GeneratedComment() string {
return fmt.Sprintf("RelSpec %s (built: %s)", Version, BuildDate)
}
const AsciiLogo = `
██████╗ ███████╗██╗ ███████╗██████╗ ███████╗ ██████╗
██╔══██╗██╔════╝██║ ██╔════╝██╔══██╗██╔════╝██╔════╝
██████╔╝█████╗ ██║ ███████╗██████╔╝█████╗ ██║
██╔══██╗██╔══╝ ██║ ╚════██║██╔═══╝ ██╔══╝ ██║
██║ ██║███████╗███████╗███████║██║ ███████╗╚██████╗
╚═╝ ╚═╝╚══════╝╚══════╝╚══════╝╚═╝ ╚══════╝ ╚═════╝
[ IN ] ──▶ [ RELSPEC ] ──▶ [ OUT ]
╔══════════════════════════════════════╗
║ ║
║ © WARKY DEVS ║
║ Author: Hein (hein@warky.dev) ║
║ ║
╚══════════════════════════════════════╝
`
+3 -5
View File
@@ -264,7 +264,7 @@ func compareColumnDetails(source, target *models.Column) map[string]any {
// comparableColumn accepts DBML's compact type/default spelling as well as
// PostgreSQL's normalized fields (for example varchar(255) vs varchar + 255).
func comparableColumn(column *models.Column) (string, int, any) {
func comparableColumn(column *models.Column) (normalizedType string, length int, defaultVal any) {
typeName := strings.TrimSpace(column.Type)
defaultValue := column.Default
lower := strings.ToLower(typeName)
@@ -274,7 +274,7 @@ func comparableColumn(column *models.Column) (string, int, any) {
}
typeName = strings.TrimSpace(typeName[:i])
}
length := column.Length
length = column.Length
if open := strings.LastIndex(typeName, "("); open >= 0 && strings.HasSuffix(typeName, ")") {
if parsed, err := strconv.Atoi(strings.TrimSpace(typeName[open+1 : len(typeName)-1])); err == nil && length == 0 {
length = parsed
@@ -353,9 +353,7 @@ func compareIndexes(source, target map[string]*models.Index) *IndexDiff {
}
for _, key := range sortedKeys(remainingTarget) {
for _, index := range remainingTarget[key] {
diff.Extra = append(diff.Extra, index)
}
diff.Extra = append(diff.Extra, remainingTarget[key]...)
}
return diff
}
-7
View File
@@ -130,7 +130,6 @@ func TestFormatSummary(t *testing.T) {
t.Run(tt.name, func(t *testing.T) {
var buf bytes.Buffer
err := formatSummary(tt.result, &buf)
if err != nil {
t.Errorf("formatSummary() error = %v", err)
return
@@ -159,7 +158,6 @@ func TestFormatJSON(t *testing.T) {
var buf bytes.Buffer
err := formatJSON(result, &buf)
if err != nil {
t.Errorf("formatJSON() error = %v", err)
return
@@ -288,7 +286,6 @@ func TestFormatHTML(t *testing.T) {
t.Run(tt.name, func(t *testing.T) {
var buf bytes.Buffer
err := formatHTML(tt.result, &buf)
if err != nil {
t.Errorf("formatHTML() error = %v", err)
return
@@ -334,7 +331,6 @@ func TestFormatSummaryWithColumns(t *testing.T) {
var buf bytes.Buffer
err := formatSummary(result, &buf)
if err != nil {
t.Errorf("formatSummary() error = %v", err)
return
@@ -383,7 +379,6 @@ func TestFormatSummaryWithIndexes(t *testing.T) {
var buf bytes.Buffer
err := formatSummary(result, &buf)
if err != nil {
t.Errorf("formatSummary() error = %v", err)
return
@@ -425,7 +420,6 @@ func TestFormatSummaryWithConstraints(t *testing.T) {
var buf bytes.Buffer
err := formatSummary(result, &buf)
if err != nil {
t.Errorf("formatSummary() error = %v", err)
return
@@ -448,7 +442,6 @@ func TestFormatJSONIndentation(t *testing.T) {
var buf bytes.Buffer
err := formatJSON(result, &buf)
if err != nil {
t.Errorf("formatJSON() error = %v", err)
return
+1
View File
@@ -123,6 +123,7 @@ rules:
| `missing_primary_key` | `have_primary_key` | Ensure tables have primary keys |
| `orphaned_foreign_key` | `orphaned_foreign_key` | Detect FKs referencing non-existent tables |
| `circular_dependency` | `circular_dependency` | Detect circular FK dependencies |
| `duplicate_index_name` | `duplicate_index_name` | Index / PK / unique names must be unique per schema (default: `enforce`) |
## Rule Configuration
+2 -1
View File
@@ -161,6 +161,7 @@ func getValidator(functionName string) (validatorFunc, bool) {
"have_primary_key": validateMissingPrimaryKey,
"orphaned_foreign_key": validateOrphanedForeignKey,
"circular_dependency": validateCircularDependency,
"duplicate_index_name": validateDuplicateIndexName,
}
fn, exists := validators[functionName]
@@ -168,7 +169,7 @@ func getValidator(functionName string) (validatorFunc, bool) {
}
// createResult is a helper to create a validation result
func createResult(ruleName string, passed bool, message string, location string, context map[string]interface{}) ValidationResult {
func createResult(ruleName string, passed bool, message, location string, context map[string]interface{}) ValidationResult {
return ValidationResult{
RuleName: ruleName,
Message: message,
-3
View File
@@ -29,7 +29,6 @@ func TestInspect(t *testing.T) {
inspector := NewInspector(db, config)
report, err := inspector.Inspect()
if err != nil {
t.Fatalf("Inspect() returned error: %v", err)
}
@@ -103,7 +102,6 @@ func TestInspectWithDisabledRules(t *testing.T) {
inspector := NewInspector(db, config)
report, err := inspector.Inspect()
if err != nil {
t.Fatalf("Inspect() with disabled rules returned error: %v", err)
}
@@ -135,7 +133,6 @@ func TestInspectWithEnforcedRules(t *testing.T) {
inspector := NewInspector(db, config)
report, err := inspector.Inspect()
if err != nil {
t.Fatalf("Inspect() returned error: %v", err)
}
+2 -2
View File
@@ -141,7 +141,7 @@ func (f *MarkdownFormatter) formatHeader(text string) string {
return f.formatBold("# " + text)
}
func (f *MarkdownFormatter) formatSubheader(text string, color string) string {
func (f *MarkdownFormatter) formatSubheader(text, color string) string {
header := "### " + text
if f.UseColors {
return color + colorBold + header + colorReset
@@ -156,7 +156,7 @@ func (f *MarkdownFormatter) formatBold(text string) string {
return "**" + text + "**"
}
func (f *MarkdownFormatter) colorize(text string, color string) string {
func (f *MarkdownFormatter) colorize(text, color string) string {
if f.UseColors {
return color + text + colorReset
}
+5
View File
@@ -154,6 +154,11 @@ func GetDefaultConfig() *Config {
Function: "circular_dependency",
Message: "Circular foreign key dependency detected",
},
"duplicate_index_name": {
Enabled: "enforce",
Function: "duplicate_index_name",
Message: "Index name is reused within the schema; PostgreSQL skips the duplicate CREATE INDEX IF NOT EXISTS",
},
},
}
}
+3 -3
View File
@@ -37,6 +37,7 @@ func TestGetDefaultConfig(t *testing.T) {
"missing_primary_key",
"orphaned_foreign_key",
"circular_dependency",
"duplicate_index_name",
}
for _, ruleName := range expectedRules {
@@ -49,7 +50,6 @@ func TestGetDefaultConfig(t *testing.T) {
func TestLoadConfig_NonExistentFile(t *testing.T) {
// Try to load a non-existent file
config, err := LoadConfig("/path/to/nonexistent/file.yaml")
if err != nil {
t.Fatalf("LoadConfig() with non-existent file returned error: %v", err)
}
@@ -83,7 +83,7 @@ rules:
message: "Table name too long"
`
err := os.WriteFile(configPath, []byte(configContent), 0644)
err := os.WriteFile(configPath, []byte(configContent), 0o644)
if err != nil {
t.Fatalf("Failed to create test config file: %v", err)
}
@@ -133,7 +133,7 @@ func TestLoadConfig_InvalidYAML(t *testing.T) {
invalidContent := `invalid: yaml: content: {[}]`
err := os.WriteFile(configPath, []byte(invalidContent), 0644)
err := os.WriteFile(configPath, []byte(invalidContent), 0o644)
if err != nil {
t.Fatalf("Failed to create test config file: %v", err)
}
+61
View File
@@ -643,3 +643,64 @@ func contains(slice []string, value string) bool {
}
return false
}
// validateDuplicateIndexName checks that index names are unique per schema.
// PostgreSQL keeps indexes in the schema-wide relation namespace, so a name
// reused on another table makes CREATE INDEX IF NOT EXISTS silently skip it.
// Primary key and unique constraints create backing indexes and share that
// namespace too. An index and a constraint with the same name on the same
// table describe one object and are not reported.
func validateDuplicateIndexName(db *models.Database, rule Rule, ruleName string) []ValidationResult {
results := []ValidationResult{}
for _, schema := range db.Schemas {
// lowercased name -> "table" entries, one per distinct object
owners := make(map[string][]string)
display := make(map[string]string)
order := []string{}
add := func(name, table string, sameTableMerges bool) {
key := strings.ToLower(name)
if _, seen := owners[key]; !seen {
order = append(order, key)
display[key] = name
}
if sameTableMerges && contains(owners[key], table) {
return
}
owners[key] = append(owners[key], table)
}
for _, table := range schema.Tables {
for _, key := range sortedKeys(table.Indexes) {
if name := table.Indexes[key].Name; name != "" {
add(name, table.Name, false)
}
}
for _, c := range sortConstraints(table.Constraints) {
if c.Name == "" || (c.Type != models.PrimaryKeyConstraint && c.Type != models.UniqueConstraint) {
continue
}
add(c.Name, table.Name, true)
}
}
for _, key := range order {
tables := owners[key]
results = append(results, createResult(
ruleName,
len(tables) == 1,
rule.Message,
formatLocation(schema.Name, display[key], "")+" on "+strings.Join(tables, ", "),
map[string]interface{}{
"schema": schema.Name,
"index": display[key],
"tables": tables,
"occurrences": len(tables),
},
))
}
}
return results
}
+81
View File
@@ -835,3 +835,84 @@ func TestFormatLocation(t *testing.T) {
})
}
}
func TestValidateDuplicateIndexName(t *testing.T) {
db := &models.Database{
Name: "testdb",
Schemas: []*models.Schema{
{
Name: "entity",
Tables: []*models.Table{
{
Name: "actor_phone",
Indexes: map[string]*models.Index{
"idx_actor": {Name: "idx_actor", Columns: []string{"rid_actor"}},
"uk_phone": {Name: "uk_phone", Columns: []string{"phone"}, Unique: true},
},
Constraints: map[string]*models.Constraint{
// Same name as the index on the same table: one object.
"uk_phone": {Name: "uk_phone", Type: models.UniqueConstraint, Columns: []string{"phone"}},
},
},
{
Name: "actor_email",
Indexes: map[string]*models.Index{
"idx_actor": {Name: "idx_actor", Columns: []string{"rid_actor"}},
},
Constraints: map[string]*models.Constraint{
"UK_Phone": {Name: "UK_Phone", Type: models.UniqueConstraint, Columns: []string{"email"}},
},
},
{
Name: "actor_address",
Indexes: map[string]*models.Index{
"idx_actor": {Name: "idx_actor", Columns: []string{"rid_actor"}},
"idx_actor#2": {Name: "idx_actor", Columns: []string{"rid_actor", "kind"}},
"idx_address": {Name: "idx_address", Columns: []string{"line1"}},
},
},
},
},
{
// Same names in another schema do not collide.
Name: "org",
Tables: []*models.Table{
{Name: "api_provider", Indexes: map[string]*models.Index{"idx_actor": {Name: "idx_actor"}}},
},
},
},
}
results := validateDuplicateIndexName(db, Rule{Message: "dup"}, "duplicate_index_name")
got := map[string]bool{}
occ := map[string]int{}
for _, r := range results {
key := r.Context["schema"].(string) + "." + r.Context["index"].(string)
got[key] = r.Passed
occ[key] = r.Context["occurrences"].(int)
}
want := map[string]struct {
passed bool
occ int
}{
"entity.idx_actor": {false, 4},
"entity.uk_phone": {false, 2},
"entity.idx_address": {true, 1},
"org.idx_actor": {true, 1},
}
if len(got) != len(want) {
t.Fatalf("got %d results %v, want %d", len(got), got, len(want))
}
for k, w := range want {
p, ok := got[k]
if !ok {
t.Errorf("missing result for %s", k)
continue
}
if p != w.passed || occ[k] != w.occ {
t.Errorf("%s: passed=%v occurrences=%d, want passed=%v occurrences=%d", k, p, occ[k], w.passed, w.occ)
}
}
}
+500 -30
View File
@@ -19,24 +19,54 @@ import (
"fmt"
"os"
"path/filepath"
"regexp"
"sort"
"strconv"
"strings"
"gopkg.in/yaml.v3"
)
// SchemaVersion is the only job-file schema version this build understands.
const SchemaVersion = 1
// CurrentSchemaVersion is the highest job-file schema version this build was
// written for. MinSchemaVersion is the oldest it still accepts. A file that
// declares a version in between loads normally; a newer version loads
// best-effort with a warning (see Load); an older-than-minimum version is a
// hard error.
const (
CurrentSchemaVersion = 1
MinSchemaVersion = 1
)
// Built-in logfile rotation policy, used when neither the job nor its file's
// defaults block sets one.
const (
defaultLogMaxSizeBytes int64 = 5 << 20 // 5 MiB
defaultLogKeep = 3
)
// Command names are a closed allow-list. Arbitrary strings are rejected.
const (
CommandConvert = "convert" // read one or more schema files, optionally merge, write one output
CommandMerge = "merge" // additive merge of two or more schema files into one output
CommandScriptsList = "scripts-list" // deterministically list SQL scripts across one or more directories
CommandScriptsExec = "scripts-exec" // execute SQL scripts across one or more directories against a live database
CommandTempl = "templ" // apply a custom Go text template to one or more schemas
CommandSplit = "split" // extract selected schemas/tables into a separate output
CommandInspect = "inspect" // validate one or more schemas against rules and write a report
CommandDiff = "diff" // compare exactly two schemas and write a differences report
)
// SupportedCommands lists every accepted command, in help order.
var SupportedCommands = []string{CommandConvert, CommandMerge, CommandScriptsList}
var SupportedCommands = []string{
CommandConvert, CommandMerge, CommandScriptsList, CommandScriptsExec,
CommandTempl, CommandSplit, CommandInspect, CommandDiff,
}
// producerCommands are commands whose output is a schema file that another job
// may consume via from_job.
var producerCommands = map[string]bool{
CommandConvert: true, CommandMerge: true, CommandSplit: true,
}
// readerFormats are the file-based input formats a job may declare (path).
var readerFormats = map[string]bool{
@@ -60,10 +90,41 @@ var writerFormats = map[string]bool{
// live database) is supported instead of writing a file.
var execOutputFormats = map[string]bool{"pgsql": true}
// singleFileFormats are output formats that emit exactly one file (as opposed
// to a directory of files). Only these are eligible for atomic temp+rename
// writes and for being consumed by another job via from_job.
var singleFileFormats = map[string]bool{
"json": true, "yaml": true, "dbml": true, "dctx": true, "drawdb": true,
"graphql": true, "pgsql": true, "mssql": true, "sqlite": true,
}
// SingleFileOutputFormat reports whether format writes exactly one file.
func SingleFileOutputFormat(format string) bool {
return singleFileFormats[strings.ToLower(format)]
}
// diffReportFormats and inspectReportFormats are the report.format values
// accepted by the diff and inspect commands respectively.
var (
diffReportFormats = map[string]bool{"summary": true, "json": true, "html": true}
inspectReportFormats = map[string]bool{"markdown": true, "json": true}
)
// File is the on-disk shape of a single job file.
type File struct {
Version int `yaml:"version"`
Jobs map[string]*Job `yaml:"jobs"`
Version int `yaml:"version"`
Defaults *Defaults `yaml:"defaults"`
Jobs map[string]*Job `yaml:"jobs"`
}
// Defaults carries file-wide settings that individual jobs may override.
type Defaults struct {
// LogMaxSize is a human-readable size ("5MB", "512KB", "1GB"). Empty
// means "use the built-in default".
LogMaxSize string `yaml:"log_max_size"`
// LogKeep is how many rotated logfiles to retain. Zero means "use the
// built-in default".
LogKeep int `yaml:"log_keep"`
}
// Job is one named job within a job file.
@@ -71,15 +132,26 @@ type Job struct {
// Name and SourceFile are populated by Load, not parsed from YAML.
Name string `yaml:"-"`
SourceFile string `yaml:"-"`
// fileDefaults is the Defaults block of the file that declared this job,
// captured by Load. nil when the file had none.
fileDefaults *Defaults `yaml:"-"`
Command string `yaml:"command"`
Description string `yaml:"description"`
DependsOn []string `yaml:"depends_on"`
Inputs []Input `yaml:"inputs"`
ScriptDirs []string `yaml:"script_dirs"`
Output *Output `yaml:"output"`
Options Options `yaml:"options"`
Logfile string `yaml:"logfile"`
Command string `yaml:"command"`
Description string `yaml:"description"`
DependsOn []string `yaml:"depends_on"`
Inputs []Input `yaml:"inputs"`
ScriptDirs []string `yaml:"script_dirs"`
Template string `yaml:"template"`
Mode string `yaml:"mode"`
FilenamePattern string `yaml:"filename_pattern"`
Output *Output `yaml:"output"`
Rules string `yaml:"rules"`
Report *Report `yaml:"report"`
Select *Select `yaml:"select"`
Options Options `yaml:"options"`
Logfile string `yaml:"logfile"`
LogMaxSize string `yaml:"log_max_size"`
LogKeep *int `yaml:"log_keep"`
}
// Input is one declared input schema.
@@ -90,6 +162,10 @@ type Input struct {
// ConnEnv is the NAME of an environment variable holding a connection
// string, used with database formats. The value is never stored here.
ConnEnv string `yaml:"conn_env"`
// FromJob names another job in the set whose file output is used as this
// input. It implies a dependency on that job. Path/Format/ConnEnv must be
// empty when FromJob is set; the format is inherited from the producer.
FromJob string `yaml:"from_job"`
}
// Output is the declared output target.
@@ -100,17 +176,123 @@ type Output struct {
Overwrite bool `yaml:"overwrite"`
}
// Report is the output target for the inspect and diff commands.
type Report struct {
// Format is the report format: diff accepts summary|json|html, inspect
// accepts markdown|json. Empty means the command's default.
Format string `yaml:"format"`
Path string `yaml:"path"`
Overwrite bool `yaml:"overwrite"`
}
// Select carries the schema/table selection for the split command.
type Select struct {
Schemas []string `yaml:"schemas"`
Tables []string `yaml:"tables"`
ExcludeSchemas []string `yaml:"exclude_schemas"`
ExcludeTables []string `yaml:"exclude_tables"`
DatabaseName string `yaml:"database_name"`
}
// LogPolicy is the resolved logfile rotation policy for a job.
type LogPolicy struct {
MaxSizeBytes int64
Keep int
}
// ResolvedLogPolicy returns the effective rotation policy: the job's own
// overrides win, then its file's defaults block, then the built-in default.
func (j *Job) ResolvedLogPolicy() LogPolicy {
p := LogPolicy{MaxSizeBytes: defaultLogMaxSizeBytes, Keep: defaultLogKeep}
if j.fileDefaults != nil {
if n, err := parseHumanSize(j.fileDefaults.LogMaxSize); err == nil && n > 0 {
p.MaxSizeBytes = n
}
if j.fileDefaults.LogKeep > 0 {
p.Keep = j.fileDefaults.LogKeep
}
}
if n, err := parseHumanSize(j.LogMaxSize); err == nil && n > 0 {
p.MaxSizeBytes = n
}
if j.LogKeep != nil && *j.LogKeep >= 0 {
p.Keep = *j.LogKeep
}
return p
}
// effectiveDeps returns the union of explicit depends_on entries and the jobs
// referenced by from_job inputs, deduplicated in stable order.
func (j *Job) effectiveDeps() []string {
seen := map[string]bool{}
var deps []string
add := func(name string) {
if name == "" || name == j.Name || seen[name] {
return
}
seen[name] = true
deps = append(deps, name)
}
for _, d := range j.DependsOn {
add(d)
}
for _, in := range j.Inputs {
add(in.FromJob)
}
return deps
}
// parseHumanSize parses a byte size such as "5MB", "512 KB", "1gb" or a bare
// byte count. An empty string returns (0, nil) so callers can fall back.
func parseHumanSize(s string) (int64, error) {
s = strings.TrimSpace(s)
if s == "" {
return 0, nil
}
upper := strings.ToUpper(s)
mult := int64(1)
// Check multi-character suffixes before the bare "B".
for _, u := range []struct {
suffix string
m int64
}{
{"KB", 1 << 10}, {"MB", 1 << 20}, {"GB", 1 << 30}, {"B", 1},
} {
if strings.HasSuffix(upper, u.suffix) {
mult = u.m
upper = strings.TrimSpace(strings.TrimSuffix(upper, u.suffix))
break
}
}
n, err := strconv.ParseFloat(upper, 64)
if err != nil {
return 0, fmt.Errorf("invalid size %q", s)
}
if n < 0 {
return 0, fmt.Errorf("negative size %q", s)
}
return int64(n * float64(mult)), nil
}
// Options carries the subset of command flags a job file may set.
type Options struct {
FlattenSchema bool `yaml:"flatten_schema"`
Schema string `yaml:"schema"`
Package string `yaml:"package"`
FlattenSchema bool `yaml:"flatten_schema"`
Schema string `yaml:"schema"`
Package string `yaml:"package"`
// Types and ArrayNullable mirror the existing convert/split writer flags.
// They are passed through only to writers that already support them.
Types string `yaml:"types"`
ArrayNullable string `yaml:"array_nullable"`
ContinueOnError bool `yaml:"continue_on_error"`
SkipRelations bool `yaml:"skip_relations"`
SkipEnums bool `yaml:"skip_enums"`
SkipViews bool `yaml:"skip_views"`
SkipDomains bool `yaml:"skip_domains"`
SkipSequences bool `yaml:"skip_sequences"`
// FullDDL makes direct pgsql output execute the full idempotent DDL instead of
// diffing against the live database first.
FullDDL bool `yaml:"full_ddl"`
SkipRelations bool `yaml:"skip_relations"`
SkipEnums bool `yaml:"skip_enums"`
SkipViews bool `yaml:"skip_views"`
SkipDomains bool `yaml:"skip_domains"`
SkipSequences bool `yaml:"skip_sequences"`
}
// Dir returns the directory that a job's relative paths resolve against:
@@ -123,6 +305,9 @@ type Set struct {
Files []string
// Jobs is keyed by job name.
Jobs map[string]*Job
// Warnings holds non-fatal load-time messages (e.g. a newer-than-known
// schema version). Callers should surface these to the user.
Warnings []string
}
// Names returns all job names in deterministic (sorted) order.
@@ -195,15 +380,35 @@ func Load(paths []string) (*Set, error) {
if err != nil {
return nil, fmt.Errorf("failed to read job file %q: %w", path, err)
}
// Peek at the version first so a newer file can be parsed leniently
// (unknown fields ignored) instead of failing outright.
var probe struct {
Version int `yaml:"version"`
}
if err := yaml.Unmarshal(data, &probe); err != nil {
return nil, fmt.Errorf("invalid job file %q: %w", path, err)
}
version := probe.Version
if version == 0 {
version = CurrentSchemaVersion
}
if version < MinSchemaVersion {
return nil, fmt.Errorf("job file %q: unsupported version %d (this build accepts %d or newer)", path, version, MinSchemaVersion)
}
strict := version <= CurrentSchemaVersion
if !strict {
set.Warnings = append(set.Warnings, fmt.Sprintf(
"job file %q declares version %d, newer than this build understands (%d); loading best-effort and ignoring unknown fields",
path, version, CurrentSchemaVersion))
}
dec := yaml.NewDecoder(strings.NewReader(string(data)))
dec.KnownFields(true)
dec.KnownFields(strict)
var f File
if err := dec.Decode(&f); err != nil {
return nil, fmt.Errorf("invalid job file %q: %w", path, err)
}
if f.Version != SchemaVersion {
return nil, fmt.Errorf("job file %q: unsupported version %d (expected %d)", path, f.Version, SchemaVersion)
}
if len(f.Jobs) == 0 {
return nil, fmt.Errorf("job file %q: no jobs defined", path)
}
@@ -216,6 +421,7 @@ func Load(paths []string) (*Set, error) {
}
job.Name = name
job.SourceFile = path
job.fileDefaults = f.Defaults
origin[name] = path
set.Jobs[name] = job
}
@@ -236,13 +442,30 @@ func (s *Set) Validate() error {
errs = append(errs, fmt.Sprintf("job %q: %s", name, msg))
}
}
// Dependency references + cycles.
// Dependency references + cycles + from_job wiring.
for _, name := range s.Names() {
for _, dep := range s.Jobs[name].DependsOn {
j := s.Jobs[name]
for _, dep := range j.DependsOn {
if _, ok := s.Jobs[dep]; !ok {
errs = append(errs, fmt.Sprintf("job %q: depends_on unknown job %q", name, dep))
}
}
for i, in := range j.Inputs {
if in.FromJob == "" {
continue
}
producer, ok := s.Jobs[in.FromJob]
if !ok {
errs = append(errs, fmt.Sprintf("job %q: input[%d] from_job references unknown job %q", name, i, in.FromJob))
continue
}
if !producerCommands[producer.Command] || producer.Output == nil ||
producer.Output.Path == "" || !SingleFileOutputFormat(producer.Output.Format) {
errs = append(errs, fmt.Sprintf(
"job %q: input[%d] from_job %q must name a convert/merge/split job that writes a single-file output",
name, i, in.FromJob))
}
}
}
if cycle := s.findCycle(); cycle != "" {
errs = append(errs, fmt.Sprintf("dependency cycle detected: %s", cycle))
@@ -258,7 +481,8 @@ func (j *Job) validate() []string {
var e []string
switch j.Command {
case CommandConvert, CommandMerge, CommandScriptsList:
case CommandConvert, CommandMerge, CommandScriptsList, CommandScriptsExec,
CommandTempl, CommandSplit, CommandInspect, CommandDiff:
case "":
e = append(e, "missing command")
return e
@@ -277,6 +501,8 @@ func (j *Job) validate() []string {
}
}
checkPath("logfile", j.Logfile)
checkPath("template", j.Template)
checkPath("rules", j.Rules)
for _, in := range j.Inputs {
checkPath("input path", in.Path)
}
@@ -286,6 +512,13 @@ func (j *Job) validate() []string {
if j.Output != nil {
checkPath("output path", j.Output.Path)
}
if j.Report != nil {
checkPath("report path", j.Report.Path)
}
if _, err := parseHumanSize(j.LogMaxSize); err != nil {
e = append(e, fmt.Sprintf("log_max_size: %v", err))
}
switch j.Command {
case CommandConvert, CommandMerge:
@@ -317,11 +550,192 @@ func (j *Job) validate() []string {
if j.Output != nil {
e = append(e, "output is not valid for command \"scripts-list\"")
}
case CommandTempl:
if len(j.Inputs) < 1 {
e = append(e, "command \"templ\" requires at least 1 input")
}
for i, in := range j.Inputs {
e = append(e, validateTemplInput(i, in)...)
}
if j.Template == "" {
e = append(e, "command \"templ\" requires template")
}
mode := strings.ToLower(j.Mode)
if mode == "" {
mode = "database"
}
switch mode {
case "database", "schema", "script", "table":
default:
e = append(e, fmt.Sprintf("command \"templ\" has unsupported mode %q (supported: database, schema, script, table)", j.Mode))
}
if len(j.ScriptDirs) > 0 {
e = append(e, "script_dirs is not valid for command \"templ\"")
}
if j.Output != nil && j.Output.ConnEnv != "" {
e = append(e, "command \"templ\" does not support database output")
}
if j.Output != nil && j.Output.Format != "" {
if !strings.EqualFold(j.Output.Format, "text") {
e = append(e, "command \"templ\" accepts only output.format: text")
}
}
case CommandSplit:
if len(j.Inputs) < 1 {
e = append(e, "command \"split\" requires at least 1 input")
}
for i, in := range j.Inputs {
e = append(e, validateInput(i, in)...)
}
if len(j.ScriptDirs) > 0 {
e = append(e, "script_dirs is not valid for command \"split\"")
}
if j.Report != nil {
e = append(e, "report is not valid for command \"split\" (use output)")
}
if j.Output == nil {
e = append(e, "missing output")
} else {
if j.Output.ConnEnv != "" {
e = append(e, "command \"split\" writes a file; output.conn_env is not supported")
}
e = append(e, validateOutput(*j.Output)...)
}
case CommandInspect:
if len(j.Inputs) < 1 {
e = append(e, "command \"inspect\" requires at least 1 input")
}
for i, in := range j.Inputs {
e = append(e, validateInput(i, in)...)
}
if len(j.ScriptDirs) > 0 {
e = append(e, "script_dirs is not valid for command \"inspect\"")
}
if j.Output != nil {
e = append(e, "output is not valid for command \"inspect\" (use report)")
}
e = append(e, validateReport(j.Report, "inspect", inspectReportFormats, "markdown")...)
case CommandDiff:
if len(j.Inputs) != 2 {
e = append(e, "command \"diff\" requires exactly 2 inputs (source, target)")
}
for i, in := range j.Inputs {
e = append(e, validateInput(i, in)...)
}
if len(j.ScriptDirs) > 0 {
e = append(e, "script_dirs is not valid for command \"diff\"")
}
if j.Output != nil {
e = append(e, "output is not valid for command \"diff\" (use report)")
}
e = append(e, validateReport(j.Report, "diff", diffReportFormats, "summary")...)
case CommandScriptsExec:
if len(j.ScriptDirs) == 0 {
e = append(e, "command \"scripts-exec\" requires at least one script_dir")
}
if len(j.Inputs) > 0 {
e = append(e, "inputs is not valid for command \"scripts-exec\"")
}
if j.Report != nil {
e = append(e, "report is not valid for command \"scripts-exec\"")
}
if j.Output == nil || j.Output.ConnEnv == "" {
e = append(e, "command \"scripts-exec\" requires output.conn_env (an environment variable name holding a connection string)")
} else {
if j.Output.Path != "" {
e = append(e, "command \"scripts-exec\" executes against a database; output.path is not supported")
}
f := strings.ToLower(j.Output.Format)
if f != "" && f != "pgsql" {
e = append(e, fmt.Sprintf("command \"scripts-exec\" only supports pgsql databases (got %q)", j.Output.Format))
}
if looksLikeSecret(j.Output.ConnEnv) {
e = append(e, "output: conn_env must be an environment variable name, not a connection string")
}
}
}
return e
}
// validateReport checks a Report block for the inspect/diff commands.
func validateReport(r *Report, cmd string, allowed map[string]bool, defFmt string) []string {
if r == nil {
return []string{fmt.Sprintf("command %q requires a report block", cmd)}
}
var e []string
f := strings.ToLower(r.Format)
if f == "" {
f = defFmt
}
if !allowed[f] {
names := make([]string, 0, len(allowed))
for k := range allowed {
names = append(names, k)
}
sort.Strings(names)
e = append(e, fmt.Sprintf("command %q report.format %q is not supported (use: %s)", cmd, r.Format, strings.Join(names, ", ")))
}
// A diff summary may be written to the log; everything else needs a path.
summaryToLog := cmd == "diff" && f == "summary"
if r.Path == "" && !summaryToLog {
e = append(e, fmt.Sprintf("command %q requires report.path", cmd))
}
return e
}
func validateTemplInput(i int, in Input) []string {
if in.FromJob != "" {
return fromJobInputShape(i, in)
}
var e []string
if in.Format == "" {
return []string{fmt.Sprintf("input[%d]: missing format", i)}
}
f := strings.ToLower(in.Format)
if f == "pgsql" {
if in.ConnEnv == "" {
e = append(e, fmt.Sprintf("input[%d]: format %q requires conn_env (an environment variable name)", i, in.Format))
}
if in.Path != "" {
e = append(e, fmt.Sprintf("input[%d]: format %q takes conn_env, not path", i, in.Format))
}
} else if readerFormats[f] {
if in.Path == "" {
e = append(e, fmt.Sprintf("input[%d]: missing path", i))
}
if in.ConnEnv != "" {
e = append(e, fmt.Sprintf("input[%d]: format %q does not use conn_env", i, in.Format))
}
} else {
e = append(e, fmt.Sprintf("input[%d]: unsupported templ input format %q", i, in.Format))
}
if looksLikeSecret(in.ConnEnv) {
e = append(e, fmt.Sprintf("input[%d]: conn_env must be an environment variable name, not a connection string", i))
}
return e
}
// fromJobInputShape checks the structural rules for an input that pulls its
// schema from another job's output. The referenced job's existence and kind
// are checked in Set.Validate, which can see the whole set.
func fromJobInputShape(i int, in Input) []string {
var e []string
if in.Path != "" {
e = append(e, fmt.Sprintf("input[%d]: from_job takes no path", i))
}
if in.Format != "" {
e = append(e, fmt.Sprintf("input[%d]: from_job inherits the producer's format; drop format", i))
}
if in.ConnEnv != "" {
e = append(e, fmt.Sprintf("input[%d]: from_job takes no conn_env", i))
}
return e
}
func validateInput(i int, in Input) []string {
if in.FromJob != "" {
return fromJobInputShape(i, in)
}
var e []string
if in.Format == "" {
e = append(e, fmt.Sprintf("input[%d]: missing format", i))
@@ -425,9 +839,65 @@ func SafeJoin(root, rel string) (string, error) {
if rp == ".." || strings.HasPrefix(rp, ".."+string(filepath.Separator)) {
return "", fmt.Errorf("path %q escapes the job file directory", rel)
}
// Symlink hardening: resolve symlinks on the root and on the deepest
// existing ancestor of the target, and require the target to still live
// inside the resolved root. This catches a symlink inside the job-file
// directory that points outside it.
realRoot, err := filepath.EvalSymlinks(absRoot)
if err != nil {
return "", fmt.Errorf("cannot resolve job file directory: %w", err)
}
realAnc, err := filepath.EvalSymlinks(deepestExistingAncestor(joined))
if err != nil {
return "", fmt.Errorf("cannot resolve path %q: %w", rel, err)
}
if realAnc != realRoot {
if r, err := filepath.Rel(realRoot, realAnc); err != nil ||
r == ".." || strings.HasPrefix(r, ".."+string(filepath.Separator)) {
return "", fmt.Errorf("path %q resolves outside the job file directory via a symlink", rel)
}
}
return joined, nil
}
var envReferencePattern = regexp.MustCompile(`\$\{([A-Za-z_][A-Za-z0-9_]*)\}`)
// ExpandEnv expands ${NAME} references using the current process environment.
// It deliberately supports only shell-independent variable references; job
// files are never passed through a shell. An unset variable is an error so a
// typo cannot silently turn into a relative path.
func ExpandEnv(value string) (string, error) {
var missing string
expanded := envReferencePattern.ReplaceAllStringFunc(value, func(reference string) string {
name := reference[2 : len(reference)-1]
resolved, ok := os.LookupEnv(name)
if !ok {
missing = name
return reference
}
return resolved
})
if missing != "" {
return "", fmt.Errorf("environment variable %q referenced by ${%s} is not set", missing, missing)
}
return expanded, nil
}
// deepestExistingAncestor returns p itself if it exists, otherwise the nearest
// existing parent directory (falling back to the filesystem root).
func deepestExistingAncestor(p string) string {
for {
if _, err := os.Lstat(p); err == nil {
return p
}
parent := filepath.Dir(p)
if parent == p {
return p
}
p = parent
}
}
// Plan returns the jobs to execute for name in dependency order. When
// includeDeps is false only the named job is returned (its declared
// dependencies are still validated to exist and be acyclic by Validate).
@@ -452,7 +922,7 @@ func (s *Set) Plan(name string, includeDeps bool) ([]*Job, error) {
}
inProgress[n] = true
j := s.Jobs[n]
deps := append([]string(nil), j.DependsOn...)
deps := j.effectiveDeps()
sort.Strings(deps)
for _, d := range deps {
if _, ok := s.Jobs[d]; !ok {
@@ -481,7 +951,7 @@ func (s *Set) findCycle() string {
dfs = func(n string) []string {
color[n] = 1
stack = append(stack, n)
deps := append([]string(nil), s.Jobs[n].DependsOn...)
deps := s.Jobs[n].effectiveDeps()
sort.Strings(deps)
for _, d := range deps {
if _, ok := s.Jobs[d]; !ok {
+218 -5
View File
@@ -56,13 +56,97 @@ func TestLoadRejectsUnknownFields(t *testing.T) {
}
}
func TestLoadRejectsBadVersion(t *testing.T) {
func TestLoadWarnsOnNewerVersion(t *testing.T) {
dir := t.TempDir()
p := filepath.Join(dir, "relspec.yml")
write(t, p, "version: 2\njobs:\n a:\n command: convert\n")
_, err := Load([]string{p})
if err == nil || !strings.Contains(err.Error(), "unsupported version") {
t.Fatalf("expected unsupported version error, got %v", err)
// A newer version loads best-effort with a warning, and unknown fields
// from the newer schema are ignored rather than rejected.
write(t, p, "version: 99\njobs:\n a:\n command: convert\n"+
" inputs:\n - path: a.dbml\n format: dbml\n"+
" output:\n format: json\n path: out.json\n"+
" future_field: whatever\n")
set, err := Load([]string{p})
if err != nil {
t.Fatalf("newer version should load, got %v", err)
}
if len(set.Warnings) == 0 {
t.Fatal("expected a warning about the newer version")
}
if err := set.Validate(); err != nil {
t.Fatalf("validate: %v", err)
}
}
func TestLoadAcceptsOmittedVersion(t *testing.T) {
dir := t.TempDir()
p := filepath.Join(dir, "relspec.yml")
write(t, p, "jobs:\n a:\n command: convert\n"+
" inputs:\n - path: a.dbml\n format: dbml\n"+
" output:\n format: json\n path: out.json\n")
set, err := Load([]string{p})
if err != nil {
t.Fatalf("omitted version should load, got %v", err)
}
if len(set.Warnings) != 0 {
t.Fatalf("omitted version should not warn, got %v", set.Warnings)
}
}
func TestLoadStillRejectsUnknownFieldsAtCurrentVersion(t *testing.T) {
dir := t.TempDir()
p := filepath.Join(dir, "relspec.yml")
write(t, p, "version: 1\njobs:\n a:\n command: convert\n bogus: true\n")
if _, err := Load([]string{p}); err == nil {
t.Fatal("expected unknown-field rejection at the current version")
}
}
func TestParseHumanSize(t *testing.T) {
cases := []struct {
in string
want int64
bad bool
}{
{"", 0, false},
{"512", 512, false},
{"512B", 512, false},
{"1KB", 1 << 10, false},
{"5MB", 5 << 20, false},
{"1gb", 1 << 30, false},
{" 2 MB ", 2 << 20, false},
{"nonsense", 0, true},
{"-1MB", 0, true},
}
for _, c := range cases {
got, err := parseHumanSize(c.in)
if c.bad {
if err == nil {
t.Errorf("parseHumanSize(%q): expected error", c.in)
}
continue
}
if err != nil {
t.Errorf("parseHumanSize(%q): %v", c.in, err)
continue
}
if got != c.want {
t.Errorf("parseHumanSize(%q) = %d, want %d", c.in, got, c.want)
}
}
}
func TestExpandEnv(t *testing.T) {
t.Setenv("RELSPEC_TEST_ROOT", "build/output")
got, err := ExpandEnv("${RELSPEC_TEST_ROOT}/schema-${RELSPEC_TEST_ROOT}")
if err != nil {
t.Fatal(err)
}
if want := "build/output/schema-build/output"; got != want {
t.Fatalf("ExpandEnv = %q, want %q", got, want)
}
if _, err := ExpandEnv("build/${RELSPEC_TEST_MISSING}"); err == nil {
t.Fatal("expected missing environment variable error")
}
}
@@ -239,6 +323,135 @@ func TestShippedExampleIsValid(t *testing.T) {
}
}
func TestFromJobWiring(t *testing.T) {
content := "version: 1\njobs:\n" +
" producer:\n command: convert\n" +
" inputs:\n - path: a.dbml\n format: dbml\n" +
" output:\n format: json\n path: build/schema.json\n" +
" consumer:\n command: convert\n" +
" inputs:\n - from_job: producer\n" +
" output:\n format: yaml\n path: build/schema.yaml\n"
set := loadOne(t, content)
if err := set.Validate(); err != nil {
t.Fatalf("validate: %v", err)
}
plan, err := set.Plan("consumer", true)
if err != nil {
t.Fatal(err)
}
if len(plan) != 2 || plan[0].Name != "producer" || plan[1].Name != "consumer" {
t.Fatalf("plan = %v, want [producer consumer]", plan)
}
}
func TestFromJobRejectsNonProducer(t *testing.T) {
content := "version: 1\njobs:\n" +
" lister:\n command: scripts-list\n script_dirs: [migrations]\n" +
" consumer:\n command: convert\n" +
" inputs:\n - from_job: lister\n" +
" output:\n format: yaml\n path: out.yaml\n"
set := loadOne(t, content)
if err := set.Validate(); err == nil || !strings.Contains(err.Error(), "from_job") {
t.Fatalf("want from_job producer error, got %v", err)
}
}
func TestFromJobRejectsUnknownJob(t *testing.T) {
content := "version: 1\njobs:\n" +
" consumer:\n command: convert\n" +
" inputs:\n - from_job: ghost\n" +
" output:\n format: yaml\n path: out.yaml\n"
set := loadOne(t, content)
if err := set.Validate(); err == nil || !strings.Contains(err.Error(), "unknown job") {
t.Fatalf("want unknown job error, got %v", err)
}
}
func TestFromJobCycleDetected(t *testing.T) {
content := "version: 1\njobs:\n" +
" a:\n command: convert\n" +
" inputs:\n - from_job: b\n" +
" output:\n format: json\n path: a.json\n" +
" b:\n command: convert\n" +
" inputs:\n - from_job: a\n" +
" output:\n format: json\n path: b.json\n"
set := loadOne(t, content)
if err := set.Validate(); err == nil || !strings.Contains(err.Error(), "cycle") {
t.Fatalf("want cycle error, got %v", err)
}
}
func TestSplitJobValidation(t *testing.T) {
set := loadOne(t, "version: 1\njobs:\n s:\n command: split\n"+
" inputs:\n - path: a.dbml\n format: dbml\n")
if err := set.Validate(); err == nil || !strings.Contains(err.Error(), "missing output") {
t.Fatalf("want missing output, got %v", err)
}
set = loadOne(t, "version: 1\njobs:\n s:\n command: split\n"+
" inputs:\n - path: a.dbml\n format: dbml\n"+
" select:\n tables: [users]\n"+
" output:\n format: json\n path: out.json\n")
if err := set.Validate(); err != nil {
t.Fatalf("expected valid split job, got %v", err)
}
}
func TestInspectJobValidation(t *testing.T) {
set := loadOne(t, "version: 1\njobs:\n i:\n command: inspect\n"+
" inputs:\n - path: a.dbml\n format: dbml\n")
if err := set.Validate(); err == nil || !strings.Contains(err.Error(), "report") {
t.Fatalf("want report required, got %v", err)
}
set = loadOne(t, "version: 1\njobs:\n i:\n command: inspect\n"+
" inputs:\n - path: a.dbml\n format: dbml\n"+
" report:\n format: json\n path: build/report.json\n")
if err := set.Validate(); err != nil {
t.Fatalf("expected valid inspect job, got %v", err)
}
}
func TestDiffJobValidation(t *testing.T) {
set := loadOne(t, "version: 1\njobs:\n d:\n command: diff\n"+
" inputs:\n - path: a.dbml\n format: dbml\n"+
" report:\n format: summary\n")
if err := set.Validate(); err == nil || !strings.Contains(err.Error(), "exactly 2 inputs") {
t.Fatalf("want exactly 2 inputs, got %v", err)
}
set = loadOne(t, "version: 1\njobs:\n d:\n command: diff\n"+
" inputs:\n - path: a.dbml\n format: dbml\n"+
" - path: b.dbml\n format: dbml\n"+
" report:\n format: summary\n")
if err := set.Validate(); err != nil {
t.Fatalf("expected valid diff job, got %v", err)
}
}
func TestScriptsExecValidation(t *testing.T) {
set := loadOne(t, "version: 1\njobs:\n x:\n command: scripts-exec\n"+
" script_dirs: [migrations]\n")
if err := set.Validate(); err == nil || !strings.Contains(err.Error(), "conn_env") {
t.Fatalf("want output.conn_env required, got %v", err)
}
set = loadOne(t, "version: 1\njobs:\n x:\n command: scripts-exec\n"+
" script_dirs: [migrations]\n"+
" output:\n conn_env: TARGET_DB_URL\n")
if err := set.Validate(); err != nil {
t.Fatalf("expected valid scripts-exec job, got %v", err)
}
}
func TestSafeJoinRejectsSymlinkEscape(t *testing.T) {
root := t.TempDir()
outside := t.TempDir()
link := filepath.Join(root, "link")
if err := os.Symlink(outside, link); err != nil {
t.Skipf("symlink not supported: %v", err)
}
if _, err := SafeJoin(root, "link/x.sql"); err == nil {
t.Fatal("expected rejection of a path escaping via a symlink")
}
}
func TestScriptsListValidation(t *testing.T) {
set := loadOne(t, "version: 1\njobs:\n s:\n command: scripts-list\n")
if err := set.Validate(); err == nil || !strings.Contains(err.Error(), "script_dir") {
+48 -9
View File
@@ -91,6 +91,45 @@ func (r *MergeResult) merge(target, source *models.Database, opts *MergeOptions)
if !opts.SkipDomains {
r.mergeDomains(target, source)
}
mergeDatabaseMetadata(target, source)
}
// mergeDatabaseMetadata adds missing metadata keys and unions []string values,
// so per-file reader state (e.g. pending DBML commented refs) survives a merge.
func mergeDatabaseMetadata(target, source *models.Database) {
if len(source.Metadata) == 0 {
return
}
if target.Metadata == nil {
target.Metadata = make(map[string]any, len(source.Metadata))
}
for key, srcVal := range source.Metadata {
tgtVal, exists := target.Metadata[key]
if !exists {
if list, ok := srcVal.([]string); ok {
srcVal = append([]string(nil), list...)
}
target.Metadata[key] = srcVal
continue
}
tgtList, tgtOK := tgtVal.([]string)
srcList, srcOK := srcVal.([]string)
if !tgtOK || !srcOK {
continue
}
seen := make(map[string]bool, len(tgtList))
for _, v := range tgtList {
seen[v] = true
}
for _, v := range srcList {
if !seen[v] {
tgtList = append(tgtList, v)
seen[v] = true
}
}
target.Metadata[key] = tgtList
}
}
func (r *MergeResult) mergeSchemaContents(target, source *models.Schema, opts *MergeOptions) {
@@ -118,7 +157,7 @@ func (r *MergeResult) mergeSchemaContents(target, source *models.Schema, opts *M
}
}
func (r *MergeResult) mergeTables(schema *models.Schema, source *models.Schema, opts *MergeOptions) {
func (r *MergeResult) mergeTables(schema, source *models.Schema, opts *MergeOptions) {
// Create map of existing tables
existingTables := make(map[string]*models.Table)
for _, table := range schema.Tables {
@@ -150,7 +189,7 @@ func (r *MergeResult) mergeTables(schema *models.Schema, source *models.Schema,
}
}
func (r *MergeResult) mergeColumns(table *models.Table, srcTable *models.Table) {
func (r *MergeResult) mergeColumns(table, srcTable *models.Table) {
// Create map of existing columns
existingColumns := make(map[string]*models.Column)
for colName := range table.Columns {
@@ -185,7 +224,7 @@ func (r *MergeResult) mergeColumns(table *models.Table, srcTable *models.Table)
}
}
func (r *MergeResult) mergeConstraints(table *models.Table, srcTable *models.Table) {
func (r *MergeResult) mergeConstraints(table, srcTable *models.Table) {
// Initialize constraints map if nil
if table.Constraints == nil {
table.Constraints = make(map[string]*models.Constraint)
@@ -208,7 +247,7 @@ func (r *MergeResult) mergeConstraints(table *models.Table, srcTable *models.Tab
}
}
func (r *MergeResult) mergeIndexes(table *models.Table, srcTable *models.Table) {
func (r *MergeResult) mergeIndexes(table, srcTable *models.Table) {
// Initialize indexes map if nil
if table.Indexes == nil {
table.Indexes = make(map[string]*models.Index)
@@ -231,7 +270,7 @@ func (r *MergeResult) mergeIndexes(table *models.Table, srcTable *models.Table)
}
}
func (r *MergeResult) mergeViews(schema *models.Schema, source *models.Schema) {
func (r *MergeResult) mergeViews(schema, source *models.Schema) {
// Create map of existing views
existingViews := make(map[string]*models.View)
for _, view := range schema.Views {
@@ -250,7 +289,7 @@ func (r *MergeResult) mergeViews(schema *models.Schema, source *models.Schema) {
}
}
func (r *MergeResult) mergeSequences(schema *models.Schema, source *models.Schema) {
func (r *MergeResult) mergeSequences(schema, source *models.Schema) {
// Create map of existing sequences
existingSequences := make(map[string]*models.Sequence)
for _, seq := range schema.Sequences {
@@ -269,7 +308,7 @@ func (r *MergeResult) mergeSequences(schema *models.Schema, source *models.Schem
}
}
func (r *MergeResult) mergeEnums(schema *models.Schema, source *models.Schema) {
func (r *MergeResult) mergeEnums(schema, source *models.Schema) {
// Create map of existing enums
existingEnums := make(map[string]*models.Enum)
for _, enum := range schema.Enums {
@@ -288,7 +327,7 @@ func (r *MergeResult) mergeEnums(schema *models.Schema, source *models.Schema) {
}
}
func (r *MergeResult) mergeRelations(schema *models.Schema, source *models.Schema) {
func (r *MergeResult) mergeRelations(schema, source *models.Schema) {
// Create map of existing relations
existingRelations := make(map[string]*models.Relationship)
for _, rel := range schema.Relations {
@@ -306,7 +345,7 @@ func (r *MergeResult) mergeRelations(schema *models.Schema, source *models.Schem
}
}
func (r *MergeResult) mergeDomains(target *models.Database, source *models.Database) {
func (r *MergeResult) mergeDomains(target, source *models.Database) {
// Create map of existing domains
existingDomains := make(map[string]*models.Domain)
for _, domain := range target.Domains {
+29
View File
@@ -721,3 +721,32 @@ func TestComplexMerge(t *testing.T) {
t.Error("Expected ukey_users_guid constraint to exist")
}
}
func TestMergeDatabases_Metadata(t *testing.T) {
target := &models.Database{Metadata: map[string]any{
"refs": []string{"a", "b"},
"name": "target",
}}
source := &models.Database{Metadata: map[string]any{
"refs": []string{"b", "c"},
"name": "source",
"extra": []string{"x"},
}}
MergeDatabases(target, source, nil)
if got := target.Metadata["refs"].([]string); strings.Join(got, ",") != "a,b,c" {
t.Errorf("refs = %v, want [a b c]", got)
}
if got := target.Metadata["name"]; got != "target" {
t.Errorf("name = %v, want target (existing scalar keys are kept)", got)
}
extra := target.Metadata["extra"].([]string)
if strings.Join(extra, ",") != "x" {
t.Errorf("extra = %v, want [x]", extra)
}
source.Metadata["extra"].([]string)[0] = "changed"
if extra[0] != "x" {
t.Error("copied slice must not alias the source")
}
}
+237
View File
@@ -0,0 +1,237 @@
package models
import (
"fmt"
"sort"
"strings"
)
// Directive is a dialect-specific instruction embedded in a source schema
// (currently DBML) that is preserved losslessly in the intermediate model and
// consumed only by the writer for its namespace. Directives are stored in the
// Metadata map of the object they apply to, under DirectivesMetadataKey.
//
// Example DBML: `@postgres: partition by RANGE (created_at)` parses to
// Directive{Namespace: "postgres", Key: "partition", Args: "partition by RANGE (created_at)"}.
type Directive struct {
// Namespace is the dialect the directive targets, e.g. "postgres" or "sqlite".
Namespace string `json:"namespace" yaml:"namespace"`
// Key is the lowercased first token of Args, used for duplicate detection
// and writer dispatch.
Key string `json:"key,omitempty" yaml:"key,omitempty"`
// Args is the verbatim argument text following the "@namespace:" prefix.
Args string `json:"args" yaml:"args"`
// Line is the 1-based source line the directive was read from, when known.
Line int `json:"line,omitempty" yaml:"line,omitempty"`
}
// DirectivesMetadataKey is the Metadata map key under which the ordered list of
// dialect directives for an object is stored.
const DirectivesMetadataKey = "directives"
// DirectiveKey derives the Key for a directive from its argument text: the
// lowercased first whitespace-delimited token.
func DirectiveKey(args string) string {
fields := strings.Fields(args)
if len(fields) == 0 {
return ""
}
return strings.ToLower(fields[0])
}
// AddDirective appends d to the directive list stored in meta. The caller is
// responsible for ensuring meta is non-nil (all Init* constructors allocate it).
// If d.Key is empty it is derived from d.Args.
func AddDirective(meta map[string]any, d Directive) {
if meta == nil {
return
}
if d.Key == "" {
d.Key = DirectiveKey(d.Args)
}
existing := GetDirectives(meta)
existing = append(existing, d)
meta[DirectivesMetadataKey] = existing
}
// GetDirectives returns the directives stored in meta, sorted deterministically
// by (Namespace, Line, Args). It tolerates both a freshly built []Directive and
// the []any of map[string]any produced by a JSON/YAML round-trip.
func GetDirectives(meta map[string]any) []Directive {
if meta == nil {
return nil
}
raw, ok := meta[DirectivesMetadataKey]
if !ok || raw == nil {
return nil
}
var out []Directive
switch v := raw.(type) {
case []Directive:
out = append(out, v...)
case []any:
for _, item := range v {
if d, ok := directiveFromAny(item); ok {
out = append(out, d)
}
}
}
sort.SliceStable(out, func(i, j int) bool {
if out[i].Namespace != out[j].Namespace {
return out[i].Namespace < out[j].Namespace
}
if out[i].Line != out[j].Line {
return out[i].Line < out[j].Line
}
return out[i].Args < out[j].Args
})
return out
}
// directiveFromAny decodes a single directive from the loosely typed forms that
// survive a JSON or YAML round-trip (map[string]any / map[any]any).
func directiveFromAny(item any) (Directive, bool) {
switch m := item.(type) {
case Directive:
return m, true
case map[string]any:
return directiveFromStringMap(m), true
case map[any]any:
sm := make(map[string]any, len(m))
for k, val := range m {
if ks, ok := k.(string); ok {
sm[ks] = val
}
}
return directiveFromStringMap(sm), true
}
return Directive{}, false
}
func directiveFromStringMap(m map[string]any) Directive {
d := Directive{}
if s, ok := m["namespace"].(string); ok {
d.Namespace = s
}
if s, ok := m["key"].(string); ok {
d.Key = s
}
if s, ok := m["args"].(string); ok {
d.Args = s
}
switch n := m["line"].(type) {
case int:
d.Line = n
case int64:
d.Line = int(n)
case float64:
d.Line = int(n)
}
if d.Key == "" {
d.Key = DirectiveKey(d.Args)
}
return d
}
// DirectivesForNamespace returns the directives in meta that target ns, in the
// deterministic order of GetDirectives.
func DirectivesForNamespace(meta map[string]any, ns string) []Directive {
all := GetDirectives(meta)
if len(all) == 0 {
return nil
}
out := make([]Directive, 0, len(all))
for _, d := range all {
if d.Namespace == ns {
out = append(out, d)
}
}
return out
}
// HasDirective reports whether meta contains a directive with the given
// namespace and key.
func HasDirective(meta map[string]any, ns, key string) bool {
for _, d := range GetDirectives(meta) {
if d.Namespace == ns && d.Key == key {
return true
}
}
return false
}
// DirectiveSpec describes a documented directive in the catalog.
type DirectiveSpec struct {
// Singleton means only one directive with this namespace/key may appear at
// a single location; a second one is a parse error.
Singleton bool
// Locations lists the location kinds the directive is valid at
// ("database", "table", "column", "index").
Locations []string
}
// Location kinds a directive may attach to.
const (
DirectiveLocationDatabase = "database"
DirectiveLocationTable = "table"
DirectiveLocationColumn = "column"
DirectiveLocationIndex = "index"
)
// DirectiveCatalog is the set of documented directives per namespace. It is used
// for strict-mode validation in readers and writers; unknown namespaces/keys are
// still preserved losslessly when strict mode is off.
var DirectiveCatalog = map[string]map[string]DirectiveSpec{
"postgres": {
"partition": {Singleton: true, Locations: []string{DirectiveLocationTable}},
"tablespace": {Singleton: true, Locations: []string{DirectiveLocationTable, DirectiveLocationIndex}},
"inherits": {Singleton: true, Locations: []string{DirectiveLocationTable}},
"with": {Singleton: false, Locations: []string{DirectiveLocationTable, DirectiveLocationIndex}},
"storage": {Singleton: true, Locations: []string{DirectiveLocationColumn}},
"compression": {Singleton: true, Locations: []string{DirectiveLocationColumn}},
"identity": {Singleton: true, Locations: []string{DirectiveLocationColumn}},
},
"sqlite": {
"without": {Singleton: true, Locations: []string{DirectiveLocationTable}},
"strict": {Singleton: true, Locations: []string{DirectiveLocationTable}},
"collate": {Singleton: true, Locations: []string{DirectiveLocationColumn}},
},
}
// LookupDirectiveSpec returns the catalog spec for a namespace/key and whether
// it is documented.
func LookupDirectiveSpec(ns, key string) (DirectiveSpec, bool) {
keys, ok := DirectiveCatalog[ns]
if !ok {
return DirectiveSpec{}, false
}
spec, ok := keys[key]
return spec, ok
}
// DirectiveLocationAllowed reports whether a documented directive may appear at
// the given location. Unknown directives (not in the catalog) are allowed
// everywhere so they can be preserved.
func DirectiveLocationAllowed(ns, key, location string) bool {
spec, ok := LookupDirectiveSpec(ns, key)
if !ok {
return true
}
for _, l := range spec.Locations {
if l == location {
return true
}
}
return false
}
// FormatDirectiveLine renders a directive back to its DBML source form, e.g.
// "@postgres: partition by RANGE (created_at)" or "@postgres(id): identity always".
func FormatDirectiveLine(d Directive, target string) string {
if target != "" {
return fmt.Sprintf("@%s(%s): %s", d.Namespace, target, d.Args)
}
return fmt.Sprintf("@%s: %s", d.Namespace, d.Args)
}
+133
View File
@@ -0,0 +1,133 @@
package models
import (
"encoding/json"
"testing"
)
func TestDirectiveKey(t *testing.T) {
cases := map[string]string{
"partition by RANGE (created_at)": "partition",
"WITHOUT ROWID": "without",
" strict ": "strict",
"": "",
}
for args, want := range cases {
if got := DirectiveKey(args); got != want {
t.Errorf("DirectiveKey(%q) = %q, want %q", args, got, want)
}
}
}
func TestAddDirectiveDerivesKey(t *testing.T) {
meta := map[string]any{}
AddDirective(meta, Directive{Namespace: "postgres", Args: "partition by RANGE (x)", Line: 2})
AddDirective(meta, Directive{Namespace: "postgres", Key: "tablespace", Args: "tablespace fast", Line: 3})
got := GetDirectives(meta)
if len(got) != 2 {
t.Fatalf("got %d directives, want 2", len(got))
}
if got[0].Key != "partition" {
t.Errorf("derived key = %q, want %q", got[0].Key, "partition")
}
if got[1].Key != "tablespace" {
t.Errorf("explicit key = %q, want %q", got[1].Key, "tablespace")
}
}
func TestAddDirectiveNilMeta(t *testing.T) {
// Must not panic.
AddDirective(nil, Directive{Namespace: "postgres", Args: "strict"})
}
func TestGetDirectivesOrdering(t *testing.T) {
meta := map[string]any{}
AddDirective(meta, Directive{Namespace: "sqlite", Args: "strict", Line: 9})
AddDirective(meta, Directive{Namespace: "postgres", Args: "with (b)", Line: 5})
AddDirective(meta, Directive{Namespace: "postgres", Args: "with (a)", Line: 5})
AddDirective(meta, Directive{Namespace: "postgres", Args: "partition by x", Line: 2})
got := GetDirectives(meta)
wantArgs := []string{"partition by x", "with (a)", "with (b)", "strict"}
if len(got) != len(wantArgs) {
t.Fatalf("got %d directives, want %d", len(got), len(wantArgs))
}
for i, w := range wantArgs {
if got[i].Args != w {
t.Errorf("directive[%d].Args = %q, want %q", i, got[i].Args, w)
}
}
}
func TestGetDirectivesTolerantDecodeAfterJSON(t *testing.T) {
meta := map[string]any{}
AddDirective(meta, Directive{Namespace: "postgres", Args: "partition by RANGE (created_at)", Line: 4})
AddDirective(meta, Directive{Namespace: "sqlite", Args: "without rowid", Line: 6})
blob, err := json.Marshal(meta)
if err != nil {
t.Fatalf("marshal: %v", err)
}
var round map[string]any
if err := json.Unmarshal(blob, &round); err != nil {
t.Fatalf("unmarshal: %v", err)
}
got := GetDirectives(round)
if len(got) != 2 {
t.Fatalf("got %d directives after JSON round-trip, want 2", len(got))
}
if got[0].Namespace != "postgres" || got[0].Key != "partition" || got[0].Line != 4 {
t.Errorf("post-JSON directive[0] = %+v", got[0])
}
if got[0].Args != "partition by RANGE (created_at)" {
t.Errorf("post-JSON args not verbatim: %q", got[0].Args)
}
if got[1].Namespace != "sqlite" || got[1].Key != "without" {
t.Errorf("post-JSON directive[1] = %+v", got[1])
}
}
func TestDirectivesForNamespaceAndHasDirective(t *testing.T) {
meta := map[string]any{}
AddDirective(meta, Directive{Namespace: "postgres", Args: "partition by x", Line: 1})
AddDirective(meta, Directive{Namespace: "sqlite", Args: "strict", Line: 2})
pg := DirectivesForNamespace(meta, "postgres")
if len(pg) != 1 || pg[0].Key != "partition" {
t.Errorf("DirectivesForNamespace(postgres) = %+v", pg)
}
if !HasDirective(meta, "sqlite", "strict") {
t.Error("HasDirective(sqlite, strict) = false, want true")
}
if HasDirective(meta, "postgres", "tablespace") {
t.Error("HasDirective(postgres, tablespace) = true, want false")
}
}
func TestDirectiveLocationAllowed(t *testing.T) {
if !DirectiveLocationAllowed("postgres", "partition", DirectiveLocationTable) {
t.Error("partition should be allowed at table level")
}
if DirectiveLocationAllowed("postgres", "partition", DirectiveLocationColumn) {
t.Error("partition should not be allowed at column level")
}
// Unknown directives are allowed everywhere so they can be preserved.
if !DirectiveLocationAllowed("postgres", "bogus", DirectiveLocationDatabase) {
t.Error("unknown key should be allowed everywhere")
}
if !DirectiveLocationAllowed("madeup", "x", DirectiveLocationTable) {
t.Error("unknown namespace should be allowed everywhere")
}
}
func TestFormatDirectiveLine(t *testing.T) {
d := Directive{Namespace: "postgres", Key: "identity", Args: "identity always"}
if got := FormatDirectiveLine(d, ""); got != "@postgres: identity always" {
t.Errorf("FormatDirectiveLine no target = %q", got)
}
if got := FormatDirectiveLine(d, "id"); got != "@postgres(id): identity always" {
t.Errorf("FormatDirectiveLine with target = %q", got)
}
}
+63 -53
View File
@@ -24,16 +24,17 @@ const (
// Database represents the complete database schema
type Database struct {
Name string `json:"name" yaml:"name"`
Description string `json:"description,omitempty" yaml:"description,omitempty" xml:"description,omitempty"`
Schemas []*Schema `json:"schemas" yaml:"schemas" xml:"schemas"`
Domains []*Domain `json:"domains,omitempty" yaml:"domains,omitempty" xml:"domains,omitempty"`
Comment string `json:"comment,omitempty" yaml:"comment,omitempty" xml:"comment,omitempty"`
DatabaseType DatabaseType `json:"database_type,omitempty" yaml:"database_type,omitempty" xml:"database_type,omitempty"`
DatabaseVersion string `json:"database_version,omitempty" yaml:"database_version,omitempty" xml:"database_version,omitempty"`
SourceFormat string `json:"source_format,omitempty" yaml:"source_format,omitempty" xml:"source_format,omitempty"` // Source Format of the database.
UpdatedAt string `json:"updatedat,omitempty" yaml:"updatedat,omitempty" xml:"updatedat,omitempty"`
GUID string `json:"guid" yaml:"guid" xml:"guid"`
Name string `json:"name" yaml:"name"`
Description string `json:"description,omitempty" yaml:"description,omitempty" xml:"description,omitempty"`
Schemas []*Schema `json:"schemas" yaml:"schemas" xml:"schemas"`
Domains []*Domain `json:"domains,omitempty" yaml:"domains,omitempty" xml:"domains,omitempty"`
Comment string `json:"comment,omitempty" yaml:"comment,omitempty" xml:"comment,omitempty"`
DatabaseType DatabaseType `json:"database_type,omitempty" yaml:"database_type,omitempty" xml:"database_type,omitempty"`
DatabaseVersion string `json:"database_version,omitempty" yaml:"database_version,omitempty" xml:"database_version,omitempty"`
SourceFormat string `json:"source_format,omitempty" yaml:"source_format,omitempty" xml:"source_format,omitempty"` // Source Format of the database.
Metadata map[string]any `json:"metadata,omitempty" yaml:"metadata,omitempty" xml:"-"`
UpdatedAt string `json:"updatedat,omitempty" yaml:"updatedat,omitempty" xml:"updatedat,omitempty"`
GUID string `json:"guid" yaml:"guid" xml:"guid"`
}
// SQLName returns the database name in lowercase for SQL compatibility.
@@ -226,22 +227,27 @@ func (d *Sequence) SQLName() string {
// Column represents a table column
type Column struct {
Name string `json:"name" yaml:"name" xml:"name"`
Description string `json:"description,omitempty" yaml:"description,omitempty" xml:"description,omitempty"`
Table string `json:"table" yaml:"table" xml:"table"`
Schema string `json:"schema" yaml:"schema" xml:"schema"`
Type string `json:"type" yaml:"type" xml:"type"`
Length int `json:"length,omitempty" yaml:"length,omitempty" xml:"length,omitempty"`
Precision int `json:"precision,omitempty" yaml:"precision,omitempty" xml:"precision,omitempty"`
Scale int `json:"scale,omitempty" yaml:"scale,omitempty" xml:"scale,omitempty"`
NotNull bool `json:"not_null" yaml:"not_null" xml:"not_null"`
Default any `json:"default,omitempty" yaml:"default,omitempty" xml:"default,omitempty"`
AutoIncrement bool `json:"auto_increment" yaml:"auto_increment" xml:"auto_increment"`
IsPrimaryKey bool `json:"is_primary_key" yaml:"is_primary_key" xml:"is_primary_key"`
Comment string `json:"comment,omitempty" yaml:"comment,omitempty" xml:"comment,omitempty"`
Collation string `json:"collation,omitempty" yaml:"collation,omitempty" xml:"collation,omitempty"`
Sequence uint `json:"sequence,omitempty" yaml:"sequence,omitempty" xml:"sequence,omitempty"`
GUID string `json:"guid" yaml:"guid" xml:"guid"`
Name string `json:"name" yaml:"name" xml:"name"`
Description string `json:"description,omitempty" yaml:"description,omitempty" xml:"description,omitempty"`
Table string `json:"table" yaml:"table" xml:"table"`
Schema string `json:"schema" yaml:"schema" xml:"schema"`
Type string `json:"type" yaml:"type" xml:"type"`
Length int `json:"length,omitempty" yaml:"length,omitempty" xml:"length,omitempty"`
Precision int `json:"precision,omitempty" yaml:"precision,omitempty" xml:"precision,omitempty"`
Scale int `json:"scale,omitempty" yaml:"scale,omitempty" xml:"scale,omitempty"`
NotNull bool `json:"not_null" yaml:"not_null" xml:"not_null"`
Default any `json:"default,omitempty" yaml:"default,omitempty" xml:"default,omitempty"`
AutoIncrement bool `json:"auto_increment" yaml:"auto_increment" xml:"auto_increment"`
IsPrimaryKey bool `json:"is_primary_key" yaml:"is_primary_key" xml:"is_primary_key"`
Comment string `json:"comment,omitempty" yaml:"comment,omitempty" xml:"comment,omitempty"`
Collation string `json:"collation,omitempty" yaml:"collation,omitempty" xml:"collation,omitempty"`
Metadata map[string]any `json:"metadata,omitempty" yaml:"metadata,omitempty" xml:"-"`
Sequence uint `json:"sequence,omitempty" yaml:"sequence,omitempty" xml:"sequence,omitempty"`
GUID string `json:"guid" yaml:"guid" xml:"guid"`
Generated bool `json:"generated,omitempty" yaml:"generated,omitempty" xml:"generated,omitempty"`
GenerationExpression string `json:"generation_expression,omitempty" yaml:"generation_expression,omitempty" xml:"generation_expression,omitempty"`
Identity bool `json:"identity,omitempty" yaml:"identity,omitempty" xml:"identity,omitempty"`
IdentityGeneration string `json:"identity_generation,omitempty" yaml:"identity_generation,omitempty" xml:"identity_generation,omitempty"` // "ALWAYS" or "BY DEFAULT"
}
// SQLName returns the column name in lowercase for SQL compatibility.
@@ -252,19 +258,20 @@ func (d *Column) SQLName() string {
// Index represents a database index for optimizing query performance.
// Indexes can be unique, partial, or include additional columns.
type Index struct {
Name string `json:"name" yaml:"name" xml:"name"`
Description string `json:"description,omitempty" yaml:"description,omitempty" xml:"description,omitempty"`
Table string `json:"table,omitempty" yaml:"table,omitempty" xml:"table,omitempty"`
Schema string `json:"schema,omitempty" yaml:"schema,omitempty" xml:"schema,omitempty"`
Columns []string `json:"columns" yaml:"columns" xml:"columns"`
Unique bool `json:"unique" yaml:"unique" xml:"unique"`
Type string `json:"type" yaml:"type" xml:"type"` // btree, hash, gin, gist, etc.
Where string `json:"where,omitempty" yaml:"where,omitempty" xml:"where,omitempty"` // partial index condition
Concurrent bool `json:"concurrent,omitempty" yaml:"concurrent,omitempty" xml:"concurrent,omitempty"`
Include []string `json:"include,omitempty" yaml:"include,omitempty" xml:"include,omitempty"` // INCLUDE columns
Comment string `json:"comment,omitempty" yaml:"comment,omitempty" xml:"comment,omitempty"`
Sequence uint `json:"sequence,omitempty" yaml:"sequence,omitempty" xml:"sequence,omitempty"`
GUID string `json:"guid" yaml:"guid" xml:"guid"`
Name string `json:"name" yaml:"name" xml:"name"`
Description string `json:"description,omitempty" yaml:"description,omitempty" xml:"description,omitempty"`
Table string `json:"table,omitempty" yaml:"table,omitempty" xml:"table,omitempty"`
Schema string `json:"schema,omitempty" yaml:"schema,omitempty" xml:"schema,omitempty"`
Columns []string `json:"columns" yaml:"columns" xml:"columns"`
Unique bool `json:"unique" yaml:"unique" xml:"unique"`
Type string `json:"type" yaml:"type" xml:"type"` // btree, hash, gin, gist, etc.
Where string `json:"where,omitempty" yaml:"where,omitempty" xml:"where,omitempty"` // partial index condition
Concurrent bool `json:"concurrent,omitempty" yaml:"concurrent,omitempty" xml:"concurrent,omitempty"`
Include []string `json:"include,omitempty" yaml:"include,omitempty" xml:"include,omitempty"` // INCLUDE columns
Comment string `json:"comment,omitempty" yaml:"comment,omitempty" xml:"comment,omitempty"`
Metadata map[string]any `json:"metadata,omitempty" yaml:"metadata,omitempty" xml:"-"`
Sequence uint `json:"sequence,omitempty" yaml:"sequence,omitempty" xml:"sequence,omitempty"`
GUID string `json:"guid" yaml:"guid" xml:"guid"`
}
// SQLName returns the index name in lowercase for SQL compatibility.
@@ -393,10 +400,11 @@ func (d *Script) SQLName() string {
// InitDatabase initializes a new Database with empty slices
func InitDatabase(name string) *Database {
return &Database{
Name: name,
Schemas: make([]*Schema, 0),
Domains: make([]*Domain, 0),
GUID: uuid.New().String(),
Name: name,
Schemas: make([]*Schema, 0),
Domains: make([]*Domain, 0),
Metadata: make(map[string]any),
GUID: uuid.New().String(),
}
}
@@ -431,22 +439,24 @@ func InitTable(name, schema string) *Table {
// InitColumn initializes a new Column
func InitColumn(name, table, schema string) *Column {
return &Column{
Name: name,
Table: table,
Schema: schema,
GUID: uuid.New().String(),
Name: name,
Table: table,
Schema: schema,
Metadata: make(map[string]any),
GUID: uuid.New().String(),
}
}
// InitIndex initializes a new Index with empty slices
func InitIndex(name, table, schema string) *Index {
return &Index{
Name: name,
Table: table,
Schema: schema,
Columns: make([]string, 0),
Include: make([]string, 0),
GUID: uuid.New().String(),
Name: name,
Table: table,
Schema: schema,
Columns: make([]string, 0),
Include: make([]string, 0),
Metadata: make(map[string]any),
GUID: uuid.New().String(),
}
}
+2 -1
View File
@@ -247,7 +247,8 @@ var extensionFunctionPrefixes = buildExtensionIndex(func(ext Extension) []string
func buildExtensionIndex(keys func(Extension) []string) map[string]string {
index := make(map[string]string)
for _, ext := range postgresExtensions {
for name := range postgresExtensions {
ext := postgresExtensions[name]
for _, key := range keys(ext) {
// Deterministic on collision: the alphabetically first extension wins.
if existing, ok := index[key]; ok && existing < ext.Name {
+2
View File
@@ -64,6 +64,8 @@ The reader recognizes the following Bun struct tags:
- `autoincrement` - Auto-increment column
- `default` - Default value
- `unique` - Unique constraint
- `generated` - Generated column marker (`Generated`; expression is not in the tag)
- `identity` - Identity column marker (`Identity`, `ALWAYS`)
- `rel` - Relationship definition
## Example Bun Model
+48
View File
@@ -0,0 +1,48 @@
package bun
import (
"os"
"path/filepath"
"testing"
"git.warky.dev/wdevs/relspecgo/pkg/readers"
)
func TestReader_GeneratedAndIdentityMarkers(t *testing.T) {
src := `package models
import "github.com/uptrace/bun"
type Person struct {
bun.BaseModel ` + "`bun:\"table:people,alias:p\"`" + `
ID int64 ` + "`bun:\"id,pk,type:bigint,autoincrement,identity\"`" + `
Seq int64 ` + "`bun:\"seq,type:bigint,scanonly,identity,notnull\"`" + `
FullName string ` + "`bun:\"full_name,type:text,scanonly,generated,nullzero\"`" + `
First string ` + "`bun:\"first,type:text,nullzero\"`" + `
}
`
path := filepath.Join(t.TempDir(), "person.go")
if err := os.WriteFile(path, []byte(src), 0o644); err != nil {
t.Fatal(err)
}
db, err := NewReader(&readers.ReaderOptions{FilePath: path}).ReadDatabase()
if err != nil {
t.Fatalf("ReadDatabase() error = %v", err)
}
cols := db.Schemas[0].Tables[0].Columns
if !cols["id"].Identity || cols["id"].IdentityGeneration != "ALWAYS" {
t.Errorf("id: identity = %v/%q, want true/ALWAYS", cols["id"].Identity, cols["id"].IdentityGeneration)
}
if !cols["seq"].Identity || cols["seq"].IdentityGeneration != "ALWAYS" {
t.Errorf("seq: identity = %v/%q, want true/ALWAYS", cols["seq"].Identity, cols["seq"].IdentityGeneration)
}
if !cols["full_name"].Generated {
t.Error("full_name should be generated")
}
if cols["first"].Generated || cols["first"].Identity {
t.Error("first must be an ordinary column")
}
}
+11 -4
View File
@@ -245,7 +245,7 @@ func (r *Reader) getReceiverType(expr ast.Expr) string {
}
// parseTableNameMethod parses a TableName() method and extracts the table and schema name
func (r *Reader) parseTableNameMethod(funcDecl *ast.FuncDecl) (tableName string, schemaName string) {
func (r *Reader) parseTableNameMethod(funcDecl *ast.FuncDecl) (tableName, schemaName string) {
if funcDecl.Body == nil {
return "", ""
}
@@ -578,7 +578,7 @@ func (r *Reader) parseIndexesFromTag(table *models.Table, column *models.Column,
}
// extractTableNameFromTag extracts table and schema from bun tag
func (r *Reader) extractTableNameFromTag(tag string) (tableName string, schemaName string) {
func (r *Reader) extractTableNameFromTag(tag string) (tableName, schemaName string) {
// Extract bun tag value
re := regexp.MustCompile(`bun:"table:([^"]+)"`)
matches := re.FindStringSubmatch(tag)
@@ -659,6 +659,13 @@ func (r *Reader) parseColumn(fieldName string, fieldType ast.Expr, tag string, s
hasExplicitNullableMarker = true
case "autoincrement":
column.AutoIncrement = true
case "generated":
// GENERATED ... STORED marker written by the Bun writer; the expression is not in the tag
column.Generated = true
case "identity":
// GENERATED ALWAYS AS IDENTITY marker written by the Bun writer
column.Identity = true
column.IdentityGeneration = "ALWAYS"
case "default":
// Default value from Bun tag (e.g., default:gen_random_uuid())
column.Default = value
@@ -712,12 +719,12 @@ func (r *Reader) parseTypeWithLength(typeStr string) (baseType string, length in
if pgsql.SupportsLength(rawBaseType) {
if _, err := fmt.Sscanf(matches[2], "%d", &length); err == nil {
baseType = pgsql.CanonicalizeBaseType(rawBaseType)
return
return baseType, length
}
}
}
return
return baseType, length
}
// goTypeToSQL maps Go types to SQL types
+62
View File
@@ -90,12 +90,74 @@ Ref: posts.user_id > users.id [delete: cascade]
- Default values (`default`)
- Inline references (`ref`)
- Standalone `Ref` blocks
- Commented cross-file refs (`// Ref:` — see below)
- Indexes and composite indexes
- Table notes and column notes
- Enums
- Dialect directives (`@postgres:` / `@sqlite:` — see below)
## Commented cross-file refs
`// Ref:` / `// ref:` lines (ignored by dbdiagram) become FKs + relationships once both ends are loaded.
| Rule | Behaviour |
|---|---|
| When | Single file / directory: end of read. `--from-list`, `merge`, jobs: after all inputs are combined |
| Match | `schema.table.column` on both sides, case-insensitive |
| Operators | `>`, `<`, `-` (parsed like `Ref:`) |
| Duplicate of an FK on the same columns | Skipped silently |
| Target not loaded | Kept pending; skipped with a warning on the final pass |
| Column type mismatch | Warning, FK still created (`serial`≈`integer`, `bigserial`≈`bigint`) |
| Pending state | `Database.Metadata["dbml.commented_refs"]` (`[]string`) |
API: `dbml.ResolveCommentedRefs(db, final bool) []string` returns warnings.
## Dialect directives
Lines of the form `@<namespace>[(<column>)]: <args>` embed database-specific
features that plain DBML cannot express (partitioning, `WITHOUT ROWID`,
tablespaces, index storage parameters, …). They are stored losslessly on the
relevant object's `Metadata` and round-trip unchanged through the DBML writer;
the PostgreSQL and SQLite writers translate the ones they understand to SQL.
```dbml
@postgres: search_path myapp
Table myapp.events {
id bigint [pk]
created_at timestamp [not null]
@postgres(id): identity always
@postgres: partition by RANGE (created_at)
@sqlite: without rowid
indexes {
(created_at) [name: 'idx_events_created']
@postgres: with (fillfactor=90)
}
}
```
| Position | Attaches to |
|----------|-------------|
| Before the first `Table {` | database |
| Table body, no `(target)` | that table |
| Table body, `(col)` target | column `col` (error if unknown) |
| Inside `indexes { }` | the most recently listed index entry |
`args` is preserved verbatim; the **key** (lowercased first token) drives
duplicate detection. Repeated directives are kept in order; catalog "singleton"
keys error on a second occurrence at the same location. All errors are
line-numbered.
`ReaderOptions.StrictDirectives` (CLI `--strict-directives`) turns an unknown
namespace or key into an error instead of preserving it silently.
See [`docs/DBML_DIRECTIVES.md`](../../../docs/DBML_DIRECTIVES.md) for the full
grammar and the supported-directive matrix.
## Notes
- Column notes `GENERATED ALWAYS AS (expr) STORED` and `GENERATED ALWAYS|BY DEFAULT AS IDENTITY` set `Generated`/`GenerationExpression` and `Identity`/`IdentityGeneration`; they are not kept as comments
- DBML is designed for database documentation and diagramming
- Schema name defaults to `public`
- Relationship cardinality is preserved
+218
View File
@@ -0,0 +1,218 @@
package dbml
import (
"fmt"
"regexp"
"strings"
"git.warky.dev/wdevs/relspecgo/pkg/models"
)
// CommentedRefsMetadataKey is the Database.Metadata key holding commented
// `// Ref:` lines not yet resolved against the model ([]string).
const CommentedRefsMetadataKey = "dbml.commented_refs"
// commentedRefRegex matches `// Ref: ...` and `// ref: ...`.
var commentedRefRegex = regexp.MustCompile(`^//\s*[Rr]ef\s*:\s*(.+)$`)
// commentedRef returns the ref body of a trimmed `// Ref:` comment line.
func commentedRef(line string) (string, bool) {
m := commentedRefRegex.FindStringSubmatch(line)
if m == nil {
return "", false
}
ref := strings.TrimSpace(m[1])
return ref, ref != ""
}
// PendingCommentedRefs returns the commented refs not yet resolved.
func PendingCommentedRefs(db *models.Database) []string {
if db == nil || db.Metadata == nil {
return nil
}
refs, _ := db.Metadata[CommentedRefsMetadataKey].([]string)
return refs
}
func setPendingCommentedRefs(db *models.Database, refs []string) {
if len(refs) == 0 {
if db.Metadata != nil {
delete(db.Metadata, CommentedRefsMetadataKey)
}
return
}
if db.Metadata == nil {
db.Metadata = make(map[string]any)
}
db.Metadata[CommentedRefsMetadataKey] = refs
}
// addPendingCommentedRef queues a ref, skipping exact repeats.
func addPendingCommentedRef(db *models.Database, ref string) {
refs := PendingCommentedRefs(db)
for _, existing := range refs {
if existing == ref {
return
}
}
setPendingCommentedRefs(db, append(refs, ref))
}
// ResolveCommentedRefs turns pending commented refs into foreign keys and
// relationships when both ends (schema.table.column) exist in db. A ref that
// matches an existing FK on the same columns is dropped as a duplicate.
// Unmatched refs stay pending unless final is set, in which case they are
// dropped with a warning. Returns human-readable warnings.
func ResolveCommentedRefs(db *models.Database, final bool) []string {
refs := PendingCommentedRefs(db)
if len(refs) == 0 {
return nil
}
var warnings []string
var pending []string
parser := &Reader{}
for _, ref := range refs {
fk := parser.parseRef(ref)
if fk == nil || len(fk.Columns) == 0 || len(fk.Columns) != len(fk.ReferencedColumns) {
warnings = append(warnings, fmt.Sprintf("skipping commented ref %q: cannot parse", ref))
continue
}
srcTable, srcCols, srcMissing := lookupColumns(db, fk.Schema, fk.Table, fk.Columns)
dstTable, dstCols, dstMissing := lookupColumns(db, fk.ReferencedSchema, fk.ReferencedTable, fk.ReferencedColumns)
if srcMissing != "" || dstMissing != "" {
if final {
missing := srcMissing
if missing == "" {
missing = dstMissing
}
warnings = append(warnings, fmt.Sprintf("skipping commented ref %q: %s not found", ref, missing))
} else {
pending = append(pending, ref)
}
continue
}
fk.Schema, fk.Table, fk.Columns = srcTable.Schema, srcTable.Name, columnNames(srcCols)
fk.ReferencedSchema, fk.ReferencedTable, fk.ReferencedColumns = dstTable.Schema, dstTable.Name, columnNames(dstCols)
if hasFKOnColumns(srcTable, fk.Columns) {
continue // already declared by an uncommented Ref or inline ref
}
if _, taken := srcTable.Constraints[fk.Name]; taken {
warnings = append(warnings, fmt.Sprintf("skipping commented ref %q: constraint %s already exists", ref, fk.Name))
continue
}
for i := range srcCols {
if !compatibleFKTypes(srcCols[i].Type, dstCols[i].Type) {
warnings = append(warnings, fmt.Sprintf("commented ref %q: type mismatch %s.%s.%s (%s) -> %s.%s.%s (%s)",
ref, srcTable.Schema, srcTable.Name, srcCols[i].Name, srcCols[i].Type,
dstTable.Schema, dstTable.Name, dstCols[i].Name, dstCols[i].Type))
}
}
if srcTable.Constraints == nil {
srcTable.Constraints = make(map[string]*models.Constraint)
}
srcTable.Constraints[fk.Name] = fk
addFKRelationship(srcTable, fk)
}
setPendingCommentedRefs(db, pending)
return warnings
}
// lookupColumns finds a table and its columns, case-insensitively. missing
// names the first object not found, or is empty.
func lookupColumns(db *models.Database, schemaName, tableName string, cols []string) (*models.Table, []*models.Column, string) {
qualified := schemaName + "." + tableName
var table *models.Table
for _, schema := range db.Schemas {
if !strings.EqualFold(schema.Name, schemaName) {
continue
}
for _, t := range schema.Tables {
if strings.EqualFold(t.Name, tableName) {
table = t
break
}
}
}
if table == nil {
return nil, nil, "table " + qualified
}
found := make([]*models.Column, 0, len(cols))
for _, name := range cols {
col := table.Columns[name]
if col == nil {
for _, c := range table.Columns {
if strings.EqualFold(c.Name, name) {
col = c
break
}
}
}
if col == nil {
return nil, nil, "column " + qualified + "." + name
}
found = append(found, col)
}
return table, found, ""
}
func columnNames(cols []*models.Column) []string {
names := make([]string, len(cols))
for i, c := range cols {
names[i] = c.Name
}
return names
}
// hasFKOnColumns reports whether table already has a foreign key over cols.
func hasFKOnColumns(table *models.Table, cols []string) bool {
for _, c := range table.Constraints {
if c.Type != models.ForeignKeyConstraint || len(c.Columns) != len(cols) {
continue
}
same := true
for i := range cols {
if !strings.EqualFold(c.Columns[i], cols[i]) {
same = false
break
}
}
if same {
return true
}
}
return false
}
// fkTypeAliases maps serial and alias spellings to their storage type.
var fkTypeAliases = map[string]string{
"smallserial": "smallint", "serial2": "smallint", "int2": "smallint",
"serial": "integer", "serial4": "integer", "int": "integer", "int4": "integer",
"bigserial": "bigint", "serial8": "bigint", "int8": "bigint",
}
// compatibleFKTypes compares column types ignoring case, length and serial
// vs. integer spelling. Unknown (empty) types are treated as compatible.
func compatibleFKTypes(a, b string) bool {
na, nb := normalizeFKType(a), normalizeFKType(b)
return na == "" || nb == "" || na == nb
}
func normalizeFKType(t string) string {
t = strings.ToLower(strings.TrimSpace(t))
if i := strings.Index(t, "("); i >= 0 {
t = strings.TrimSpace(t[:i])
}
if alias, ok := fkTypeAliases[t]; ok {
return alias
}
return t
}
+260
View File
@@ -0,0 +1,260 @@
package dbml
import (
"os"
"path/filepath"
"strings"
"testing"
"git.warky.dev/wdevs/relspecgo/pkg/models"
"git.warky.dev/wdevs/relspecgo/pkg/readers"
)
func readDBMLString(t *testing.T, content string) *models.Database {
t.Helper()
path := filepath.Join(t.TempDir(), "in.dbml")
if err := os.WriteFile(path, []byte(content), 0o644); err != nil {
t.Fatalf("write fixture: %v", err)
}
db, err := NewReader(&readers.ReaderOptions{FilePath: path}).ReadDatabase()
if err != nil {
t.Fatalf("ReadDatabase() error = %v", err)
}
return db
}
func findTable(db *models.Database, schema, table string) *models.Table {
for _, s := range db.Schemas {
if s.Name != schema {
continue
}
for _, t := range s.Tables {
if t.Name == table {
return t
}
}
}
return nil
}
func fkOn(table *models.Table, col string) *models.Constraint {
for _, c := range table.Constraints {
if c.Type == models.ForeignKeyConstraint && len(c.Columns) == 1 && c.Columns[0] == col {
return c
}
}
return nil
}
func relFor(table *models.Table, fkName string) *models.Relationship {
for _, r := range table.Relationships {
if r.ForeignKey == fkName {
return r
}
}
return nil
}
func TestCommentedRef(t *testing.T) {
tests := []struct {
line string
want string
ok bool
}{
{`// Ref: a.b.c > d.e.f`, `a.b.c > d.e.f`, true},
{`// ref: a.b.c - d.e.f`, `a.b.c - d.e.f`, true},
{`//Ref:a.b.c > d.e.f`, `a.b.c > d.e.f`, true},
{`// Reference notes`, "", false},
{`// see Ref: a.b.c > d.e.f`, "", false},
{`// Ref:`, "", false},
}
for _, tt := range tests {
got, ok := commentedRef(tt.line)
if ok != tt.ok || got != tt.want {
t.Errorf("commentedRef(%q) = (%q, %v), want (%q, %v)", tt.line, got, ok, tt.want, tt.ok)
}
}
}
// A commented ref whose tables are in the same file resolves on read.
func TestReader_CommentedRefSameFile(t *testing.T) {
db := readDBMLString(t, `Table "org"."department" {
"id_department" bigserial [pk]
}
Table "entity"."employee" {
"id_employee" bigserial [pk]
"rid_department" bigint
}
// Ref: "entity"."employee"."rid_department" > "org"."department"."id_department" [delete: restrict, update: restrict]
`)
emp := findTable(db, "entity", "employee")
fk := fkOn(emp, "rid_department")
if fk == nil {
t.Fatal("expected FK on rid_department")
}
if fk.ReferencedSchema != "org" || fk.ReferencedTable != "department" || fk.ReferencedColumns[0] != "id_department" {
t.Errorf("FK target = %s.%s.%v", fk.ReferencedSchema, fk.ReferencedTable, fk.ReferencedColumns)
}
if fk.OnDelete != "restrict" || fk.OnUpdate != "restrict" {
t.Errorf("FK actions = %q/%q, want restrict/restrict", fk.OnDelete, fk.OnUpdate)
}
if relFor(emp, fk.Name) == nil {
t.Error("expected relationship for FK")
}
if refs := PendingCommentedRefs(db); len(refs) != 0 {
t.Errorf("pending = %v, want none", refs)
}
}
// A cross-file commented ref stays pending after a single-file read.
func TestReader_CommentedRefCrossFilePending(t *testing.T) {
db := readDBMLString(t, `Table "entity"."employee" {
"id_employee" bigserial [pk]
"rid_department" bigint
}
// Ref: "entity"."employee"."rid_department" > "org"."department"."id_department"
`)
if fk := fkOn(findTable(db, "entity", "employee"), "rid_department"); fk != nil {
t.Fatal("FK must not resolve without the target table")
}
if refs := PendingCommentedRefs(db); len(refs) != 1 {
t.Fatalf("pending = %v, want 1", refs)
}
}
// Directory reads resolve commented refs after all files are merged.
func TestReader_CommentedRefDirectory(t *testing.T) {
db, err := NewReader(&readers.ReaderOptions{
FilePath: filepath.Join("..", "..", "..", "tests", "assets", "dbml", "multifile"),
}).ReadDatabase()
if err != nil {
t.Fatalf("ReadDatabase() error = %v", err)
}
fk := fkOn(findTable(db, "public", "posts"), "user_id")
if fk == nil {
t.Fatal("expected FK posts.user_id from 9_refs.dbml commented ref")
}
if fk.ReferencedTable != "users" || fk.OnDelete != "CASCADE" {
t.Errorf("FK = %s ondelete %s, want users ondelete CASCADE", fk.ReferencedTable, fk.OnDelete)
}
}
func crossFileDB(t *testing.T, refs ...string) *models.Database {
t.Helper()
db := readDBMLString(t, `Table "org"."department" {
"id_department" bigserial [pk]
}
Table "entity"."employee" {
"id_employee" bigserial [pk]
"rid_department" bigint
"rid_manager" integer
"rid_team" bigint
}
`)
setPendingCommentedRefs(db, refs)
return db
}
func TestResolveCommentedRefs(t *testing.T) {
t.Run("lowercase ref and one-to-one", func(t *testing.T) {
db := crossFileDB(t,
`"entity"."employee"."rid_department" > "org"."department"."id_department"`,
`entity.employee.rid_team - org.department.id_department`,
)
if w := ResolveCommentedRefs(db, true); len(w) != 0 {
t.Errorf("warnings = %v, want none", w)
}
emp := findTable(db, "entity", "employee")
for _, col := range []string{"rid_department", "rid_team"} {
fk := fkOn(emp, col)
if fk == nil {
t.Fatalf("expected FK on %s", col)
}
if relFor(emp, fk.Name) == nil {
t.Errorf("expected relationship for %s", fk.Name)
}
}
if len(emp.Relationships) != 2 {
t.Errorf("relationships = %d, want 2 (same target must not overwrite)", len(emp.Relationships))
}
})
t.Run("type mismatch warns but resolves", func(t *testing.T) {
db := crossFileDB(t, `entity.employee.rid_manager > org.department.id_department`)
w := ResolveCommentedRefs(db, true)
if len(w) != 1 || !strings.Contains(w[0], "type mismatch") {
t.Errorf("warnings = %v, want one type mismatch", w)
}
if fkOn(findTable(db, "entity", "employee"), "rid_manager") == nil {
t.Error("expected FK on rid_manager")
}
})
t.Run("missing target stays pending until final", func(t *testing.T) {
db := crossFileDB(t, `entity.employee.rid_team > hr.team.id_team`)
if w := ResolveCommentedRefs(db, false); len(w) != 0 {
t.Errorf("non-final warnings = %v, want none", w)
}
if len(PendingCommentedRefs(db)) != 1 {
t.Fatal("ref should stay pending")
}
w := ResolveCommentedRefs(db, true)
if len(w) != 1 || !strings.Contains(w[0], "table hr.team not found") {
t.Errorf("final warnings = %v, want missing table", w)
}
if len(PendingCommentedRefs(db)) != 0 {
t.Error("final pass must clear pending refs")
}
})
t.Run("missing column", func(t *testing.T) {
db := crossFileDB(t, `entity.employee.rid_nope > org.department.id_department`)
w := ResolveCommentedRefs(db, true)
if len(w) != 1 || !strings.Contains(w[0], "column entity.employee.rid_nope not found") {
t.Errorf("warnings = %v, want missing column", w)
}
})
t.Run("deduplicates against uncommented ref", func(t *testing.T) {
db := readDBMLString(t, `Table "org"."department" {
"id_department" bigserial [pk]
}
Table "entity"."employee" {
"id_employee" bigserial [pk]
"rid_department" bigint
}
Ref: "entity"."employee"."rid_department" > "org"."department"."id_department"
// Ref: "entity"."employee"."rid_department" > "org"."department"."id_department"
`)
emp := findTable(db, "entity", "employee")
count := 0
for _, c := range emp.Constraints {
if c.Type == models.ForeignKeyConstraint {
count++
}
}
if count != 1 || len(emp.Relationships) != 1 {
t.Errorf("FKs = %d, relationships = %d, want 1 and 1", count, len(emp.Relationships))
}
})
}
func TestCompatibleFKTypes(t *testing.T) {
tests := []struct {
a, b string
want bool
}{
{"bigint", "bigserial", true},
{"integer", "serial", true},
{"INT4", "integer", true},
{"varchar(10)", "varchar(20)", true},
{"bigint", "serial", false},
{"uuid", "bigint", false},
{"", "bigint", true},
}
for _, tt := range tests {
if got := compatibleFKTypes(tt.a, tt.b); got != tt.want {
t.Errorf("compatibleFKTypes(%q, %q) = %v, want %v", tt.a, tt.b, got, tt.want)
}
}
}
+137
View File
@@ -0,0 +1,137 @@
package dbml
import (
"fmt"
"regexp"
"strings"
"git.warky.dev/wdevs/relspecgo/pkg/models"
)
// directiveLineRegex matches a dialect directive line:
//
// @postgres: partition by RANGE (created_at)
// @postgres(id): identity always
//
// Group 1 is the namespace, group 2 the optional (column) target, group 3 the
// raw argument text (validated separately so error messages can be specific).
var directiveLineRegex = regexp.MustCompile(`^@([^():]*)(?:\(([^()]*)\))?\s*:(.*)$`)
// namespaceRegex is the grammar for a directive namespace.
var namespaceRegex = regexp.MustCompile(`^[a-z][a-z0-9_]*$`)
// parsedDirective is a directive line that has been parsed but not yet attached
// to a model object.
type parsedDirective struct {
namespace string
target string // column name; "" when absent
args string
line int
}
// parseDirectiveLine parses a single "@namespace[(target)]: args" line.
func parseDirectiveLine(line string, lineNo int) (parsedDirective, error) {
m := directiveLineRegex.FindStringSubmatch(line)
if m == nil {
return parsedDirective{}, fmt.Errorf(
"dbml: line %d: malformed directive %q (expected \"@namespace: args\")", lineNo, line)
}
ns := strings.TrimSpace(m[1])
target := strings.TrimSpace(m[2])
args := strings.TrimSpace(m[3])
if !namespaceRegex.MatchString(ns) {
return parsedDirective{}, fmt.Errorf(
"dbml: line %d: invalid directive namespace %q (must match [a-z][a-z0-9_]*)", lineNo, ns)
}
if args == "" {
return parsedDirective{}, fmt.Errorf("dbml: line %d: directive @%s has no arguments", lineNo, ns)
}
if target != "" {
target = stripQuotes(target)
}
return parsedDirective{namespace: ns, target: target, args: args, line: lineNo}, nil
}
// attachDirective resolves the target model object from the current parser state
// and stores the directive in its Metadata, enforcing location, duplicate and
// strict-mode rules.
func (r *Reader) attachDirective(
pd parsedDirective,
db *models.Database,
table *models.Table,
inTable, inIndexes bool,
lastIndex *models.Index,
) error {
strict := r.options != nil && r.options.StrictDirectives
key := models.DirectiveKey(pd.args)
var meta map[string]any
var location string
switch {
case inIndexes:
if pd.target != "" {
return fmt.Errorf("dbml: line %d: directive target (%s) is not allowed inside an indexes block", pd.line, pd.target)
}
if lastIndex == nil {
return fmt.Errorf("dbml: line %d: directive @%s must follow an index definition", pd.line, pd.namespace)
}
if lastIndex.Metadata == nil {
lastIndex.Metadata = make(map[string]any)
}
meta = lastIndex.Metadata
location = models.DirectiveLocationIndex
case inTable && table != nil:
if pd.target != "" {
col, ok := table.Columns[pd.target]
if !ok {
return fmt.Errorf("dbml: line %d: directive target column %q not found in table %q", pd.line, pd.target, table.Name)
}
if col.Metadata == nil {
col.Metadata = make(map[string]any)
}
meta = col.Metadata
location = models.DirectiveLocationColumn
} else {
if table.Metadata == nil {
table.Metadata = make(map[string]any)
}
meta = table.Metadata
location = models.DirectiveLocationTable
}
default:
if pd.target != "" {
return fmt.Errorf("dbml: line %d: directive target (%s) is only valid inside a table", pd.line, pd.target)
}
if db.Metadata == nil {
db.Metadata = make(map[string]any)
}
meta = db.Metadata
location = models.DirectiveLocationDatabase
}
spec, documented := models.LookupDirectiveSpec(pd.namespace, key)
if strict && !documented {
return fmt.Errorf("dbml: line %d: unknown directive @%s: %s (strict mode)", pd.line, pd.namespace, key)
}
if documented && !models.DirectiveLocationAllowed(pd.namespace, key, location) {
return fmt.Errorf("dbml: line %d: directive @%s: %s is not valid at %s level", pd.line, pd.namespace, key, location)
}
if documented && spec.Singleton && models.HasDirective(meta, pd.namespace, key) {
return fmt.Errorf("dbml: line %d: duplicate @%s directive %q at %s level", pd.line, pd.namespace, key, location)
}
models.AddDirective(meta, models.Directive{
Namespace: pd.namespace,
Key: key,
Args: pd.args,
Line: pd.line,
})
return nil
}
+181
View File
@@ -0,0 +1,181 @@
package dbml
import (
"strings"
"testing"
"git.warky.dev/wdevs/relspecgo/pkg/models"
"git.warky.dev/wdevs/relspecgo/pkg/readers"
)
func parse(t *testing.T, strict bool, src string) (*models.Database, error) {
t.Helper()
r := NewReader(&readers.ReaderOptions{StrictDirectives: strict})
return r.parseDBML(src)
}
func firstTable(t *testing.T, db *models.Database) *models.Table {
t.Helper()
if len(db.Schemas) == 0 || len(db.Schemas[0].Tables) == 0 {
t.Fatal("no table parsed")
}
return db.Schemas[0].Tables[0]
}
func TestDirectives_AttachAtEachLocation(t *testing.T) {
src := `@postgres: search_path myapp
Table myapp.events {
id bigint [pk]
created_at timestamp [not null]
@postgres(id): identity always
@postgres: partition by RANGE (created_at)
indexes {
(created_at) [name: 'idx_events_created']
@postgres: with (fillfactor=90)
}
}
`
db, err := parse(t, false, src)
if err != nil {
t.Fatalf("parse: %v", err)
}
if !models.HasDirective(db.Metadata, "postgres", "search_path") {
t.Errorf("database-level directive missing: %+v", db.Metadata)
}
tbl := firstTable(t, db)
if !models.HasDirective(tbl.Metadata, "postgres", "partition") {
t.Errorf("table-level directive missing: %+v", tbl.Metadata)
}
col := tbl.Columns["id"]
if col == nil || !models.HasDirective(col.Metadata, "postgres", "identity") {
t.Errorf("column-level directive missing")
}
// Verbatim args preserved.
if d := models.DirectivesForNamespace(col.Metadata, "postgres"); len(d) != 1 || d[0].Args != "identity always" {
t.Errorf("column directive args = %+v", d)
}
var idx *models.Index
for _, i := range tbl.Indexes {
idx = i
}
if idx == nil || !models.HasDirective(idx.Metadata, "postgres", "with") {
t.Errorf("index-level directive missing: %+v", idx)
}
}
func TestDirectives_RepeatablePreservedAndOrdered(t *testing.T) {
src := `Table s.t {
id int [pk]
@postgres: with (fillfactor=90)
@postgres: with (autovacuum_enabled=off)
}
`
db, err := parse(t, false, src)
if err != nil {
t.Fatalf("parse: %v", err)
}
tbl := firstTable(t, db)
got := models.DirectivesForNamespace(tbl.Metadata, "postgres")
if len(got) != 2 {
t.Fatalf("got %d directives, want 2", len(got))
}
if got[0].Args != "with (fillfactor=90)" || got[1].Args != "with (autovacuum_enabled=off)" {
t.Errorf("repeatable directives out of order: %+v", got)
}
}
func TestDirectives_SingletonDuplicateErrors(t *testing.T) {
src := `Table s.t {
id int [pk]
@postgres: partition by RANGE (a)
@postgres: partition by LIST (b)
}
`
_, err := parse(t, false, src)
if err == nil || !strings.Contains(err.Error(), "duplicate") {
t.Fatalf("want duplicate error, got %v", err)
}
if !strings.Contains(err.Error(), "line 4") {
t.Errorf("error not line-numbered: %v", err)
}
}
func TestDirectives_MalformedErrors(t *testing.T) {
cases := map[string]string{
"no colon": "@postgres partition by x",
"empty args": "@postgres:",
"bad namespace": "@Postgres: partition by x",
"numeric prefix": "@1x: foo",
}
for name, line := range cases {
t.Run(name, func(t *testing.T) {
src := "Table s.t {\n id int [pk]\n " + line + "\n}\n"
_, err := parse(t, false, src)
if err == nil {
t.Fatalf("want error for %q", line)
}
if !strings.Contains(err.Error(), "line 3") {
t.Errorf("error not line-numbered: %v", err)
}
})
}
}
func TestDirectives_UnknownPreservedNonStrict(t *testing.T) {
src := `Table s.t {
id int [pk]
@postgres: frobnicate all the things
@clickhouse: engine MergeTree
}
`
db, err := parse(t, false, src)
if err != nil {
t.Fatalf("parse: %v", err)
}
tbl := firstTable(t, db)
if !models.HasDirective(tbl.Metadata, "postgres", "frobnicate") {
t.Error("unknown postgres key not preserved")
}
if !models.HasDirective(tbl.Metadata, "clickhouse", "engine") {
t.Error("unknown namespace not preserved")
}
}
func TestDirectives_StrictErrors(t *testing.T) {
src := `Table s.t {
id int [pk]
@postgres: frobnicate x
}
`
_, err := parse(t, true, src)
if err == nil || !strings.Contains(err.Error(), "strict mode") {
t.Fatalf("want strict-mode error, got %v", err)
}
}
func TestDirectives_UnknownColumnTargetErrors(t *testing.T) {
src := `Table s.t {
id int [pk]
@postgres(missing): identity always
}
`
_, err := parse(t, false, src)
if err == nil || !strings.Contains(err.Error(), "not found") {
t.Fatalf("want unknown-column error, got %v", err)
}
}
func TestDirectives_WrongLocationErrors(t *testing.T) {
// partition is table-only.
src := "@postgres: partition by RANGE (x)\n\nTable s.t {\n id int [pk]\n}\n"
_, err := parse(t, false, src)
if err == nil || !strings.Contains(err.Error(), "not valid at database level") {
t.Fatalf("want location error, got %v", err)
}
}
+38
View File
@@ -0,0 +1,38 @@
package dbml
import (
"regexp"
"strings"
"git.warky.dev/wdevs/relspecgo/pkg/models"
)
var (
generatedNoteRegex = regexp.MustCompile(`(?is)^GENERATED\s+ALWAYS\s+AS\s*\((.*)\)\s*STORED$`)
identityNoteRegex = regexp.MustCompile(`(?i)^GENERATED\s+(ALWAYS|BY\s+DEFAULT)\s+AS\s+IDENTITY$`)
)
// applyColumnMarkerNote interprets a column note written by the DBML writer for
// GENERATED ... STORED and GENERATED ... AS IDENTITY columns, setting the matching
// model fields. It returns true when the note was a marker, in which case it is not
// a user comment.
func applyColumnMarkerNote(column *models.Column, rawNote string) bool {
note := strings.TrimSpace(rawNote)
if len(note) >= 2 && (note[0] == '\'' || note[0] == '"') && note[len(note)-1] == note[0] {
note = note[1 : len(note)-1]
}
note = strings.ReplaceAll(strings.ReplaceAll(note, `\'`, `'`), `\\`, `\`)
note = strings.TrimSpace(note)
if m := generatedNoteRegex.FindStringSubmatch(note); m != nil {
column.Generated = true
column.GenerationExpression = strings.TrimSpace(m[1])
return true
}
if m := identityNoteRegex.FindStringSubmatch(note); m != nil {
column.Identity = true
column.IdentityGeneration = strings.Join(strings.Fields(strings.ToUpper(m[1])), " ")
return true
}
return false
}
+223 -30
View File
@@ -48,7 +48,22 @@ func (r *Reader) ReadDatabase() (*models.Database, error) {
return nil, fmt.Errorf("failed to read file: %w", err)
}
return r.parseDBML(string(content))
db, err := r.parseDBML(string(content))
if err != nil {
return nil, err
}
r.resolveCommentedRefs(db)
return db, nil
}
// resolveCommentedRefs resolves the commented refs whose tables are loaded.
// Unmatched refs stay pending for a later pass over a combined model.
func (r *Reader) resolveCommentedRefs(db *models.Database) {
for _, w := range ResolveCommentedRefs(db, false) {
if r.options.Progress != nil {
r.options.Progress("warning: " + w)
}
}
}
// ReadSchema reads and parses DBML input, returning a Schema model
@@ -125,6 +140,7 @@ func (r *Reader) readDirectoryDBML(dirPath string) (*models.Database, error) {
}
}
r.resolveCommentedRefs(db)
return db, nil
}
@@ -305,9 +321,39 @@ func sortDBMLFiles(files []string) []string {
// Merges: Columns (map), Constraints (map), Indexes (map), Relationships (map)
// Uses first non-empty Description
func mergeTable(baseTable, fileTable *models.Table) {
// Merge columns (map naturally merges - later keys overwrite)
for key, col := range fileTable.Columns {
baseTable.Columns[key] = col
// Merge columns. Each file numbers its own columns from 1, so a table split
// across files would otherwise end up with colliding Column.Sequence values
// and writers would fall back to alphabetical order. Re-base the incoming
// file's new columns after the highest sequence already present, preserving
// their in-file order. Columns that overwrite an existing key keep the
// original position.
var maxSeq uint
for _, col := range baseTable.Columns {
if col.Sequence > maxSeq {
maxSeq = col.Sequence
}
}
incoming := make([]*models.Column, 0, len(fileTable.Columns))
for _, col := range fileTable.Columns {
incoming = append(incoming, col)
}
sort.Slice(incoming, func(i, j int) bool {
if incoming[i].Sequence != incoming[j].Sequence {
return incoming[i].Sequence < incoming[j].Sequence
}
return incoming[i].Name < incoming[j].Name
})
var added uint
for _, col := range incoming {
if existing, ok := baseTable.Columns[col.Name]; ok {
col.Sequence = existing.Sequence
} else {
added++
col.Sequence = maxSeq + added
}
baseTable.Columns[col.Name] = col
}
// Merge constraints
@@ -410,6 +456,10 @@ func mergeDatabase(baseDB, fileDB *models.Database) {
// Merge domains
baseDB.Domains = append(baseDB.Domains, fileDB.Domains...)
for _, ref := range PendingCommentedRefs(fileDB) {
addPendingCommentedRef(baseDB, ref)
}
// Use first non-empty description
if baseDB.Description == "" && fileDB.Description != "" {
baseDB.Description = fileDB.Description
@@ -434,16 +484,55 @@ func (r *Reader) parseDBML(content string) (*models.Database, error) {
var currentSchema string
var inIndexes bool
var inTable bool
var inTableNote bool
var tableNoteLines []string
tableNoteStartLine := 0
var columnSeq uint
var lastIndex *models.Index // most recent index in the current Indexes block
lineNo := 0
tableRegex := regexp.MustCompile(`^Table\s+(.+?)\s*{`)
refRegex := regexp.MustCompile(`^Ref:\s+(.+)`)
for scanner.Scan() {
line := strings.TrimSpace(scanner.Text())
lineNo++
rawLine := scanner.Text()
line := strings.TrimSpace(rawLine)
// Skip empty lines and comments
// A table note can use DBML's triple-quoted form. Its contents must be
// consumed before normal parsing, otherwise each prose line is mistaken
// for a column declaration.
if inTableNote {
if line == "'''" {
setTableNote(currentTable, strings.TrimSpace(strings.Join(tableNoteLines, "\n")))
inTableNote = false
tableNoteLines = nil
continue
}
tableNoteLines = append(tableNoteLines, strings.TrimSpace(rawLine))
continue
}
// Skip empty lines and comments. A commented `// Ref:` is kept as a
// pending cross-file ref, resolved once every file is loaded.
if line == "" || strings.HasPrefix(line, "//") {
if ref, ok := commentedRef(line); ok {
addPendingCommentedRef(db, ref)
}
continue
}
// Parse a dialect directive (@postgres:, @sqlite:, …). Handled before
// table/column/index parsing so directive lines are never mistaken for
// columns.
if strings.HasPrefix(line, "@") {
pd, err := parseDirectiveLine(line, lineNo)
if err != nil {
return nil, err
}
if err := r.attachDirective(pd, db, currentTable, inTable, inIndexes, lastIndex); err != nil {
return nil, err
}
continue
}
@@ -474,8 +563,10 @@ func (r *Reader) parseDBML(content string) (*models.Database, error) {
continue
}
// End of table definition
if inTable && line == "}" {
// End of table definition. Guarded by !inIndexes so the closing brace
// of an `indexes { }` block is not mistaken for the end of the table
// (which would drop any table-level content that follows it).
if inTable && !inIndexes && line == "}" {
if currentTable != nil && currentSchema != "" {
schemaMap[currentSchema].Tables = append(schemaMap[currentSchema].Tables, currentTable)
currentTable = nil
@@ -488,12 +579,14 @@ func (r *Reader) parseDBML(content string) (*models.Database, error) {
// Parse indexes section
if inTable && (strings.HasPrefix(line, "Indexes {") || strings.HasPrefix(line, "indexes {")) {
inIndexes = true
lastIndex = nil
continue
}
// End of indexes section
if inIndexes && line == "}" {
inIndexes = false
lastIndex = nil
continue
}
@@ -512,16 +605,30 @@ func (r *Reader) parseDBML(content string) (*models.Database, error) {
index := r.parseIndex(line, currentTable.Name, currentSchema)
if index != nil {
currentTable.Indexes[index.Name] = index
// Keep a reused name under a unique map key so the duplicate is
// not silently dropped; the inspector reports it.
key := index.Name
for n := 2; currentTable.Indexes[key] != nil; n++ {
key = fmt.Sprintf("%s#%d", index.Name, n)
}
currentTable.Indexes[key] = index
lastIndex = index
}
continue
}
// Parse table note
if inTable && currentTable != nil && strings.HasPrefix(line, "Note:") {
note := strings.TrimPrefix(line, "Note:")
// Parse table note. DBML files in the wild use both `Note:` and
// `note:`, so accept either spelling.
if inTable && currentTable != nil && strings.HasPrefix(strings.ToLower(line), "note:") {
note := strings.TrimSpace(line[len("note:"):])
if strings.TrimSpace(note) == "'''" {
inTableNote = true
tableNoteLines = nil
tableNoteStartLine = lineNo
continue
}
note = strings.Trim(note, " '\"")
currentTable.Description = note
setTableNote(currentTable, note)
continue
}
@@ -558,6 +665,13 @@ func (r *Reader) parseDBML(content string) (*models.Database, error) {
}
}
if err := scanner.Err(); err != nil {
return nil, fmt.Errorf("failed to scan DBML: %w", err)
}
if inTableNote {
return nil, fmt.Errorf("dbml: line %d: unterminated triple-quoted table note", tableNoteStartLine)
}
// Assign pending constraints to their respective tables
for _, constraint := range pendingConstraints {
// Find the table this constraint belongs to
@@ -575,20 +689,12 @@ func (r *Reader) parseDBML(content string) (*models.Database, error) {
// for DBML refs so diffing equivalent schemas compares the same model.
for _, schema := range schemaMap {
for _, table := range schema.Tables {
for _, constraint := range table.Constraints {
for _, name := range sortedConstraintNames(table.Constraints) {
constraint := table.Constraints[name]
if constraint.Type != models.ForeignKeyConstraint {
continue
}
name := fmt.Sprintf("%s_to_%s", table.Name, constraint.ReferencedTable)
relationship := models.InitRelationship(name, models.OneToMany)
relationship.FromTable = table.Name
relationship.FromSchema = table.Schema
relationship.FromColumns = append([]string(nil), constraint.Columns...)
relationship.ToTable = constraint.ReferencedTable
relationship.ToSchema = constraint.ReferencedSchema
relationship.ToColumns = append([]string(nil), constraint.ReferencedColumns...)
relationship.ForeignKey = constraint.Name
table.Relationships[name] = relationship
addFKRelationship(table, constraint)
}
}
}
@@ -601,6 +707,53 @@ func (r *Reader) parseDBML(content string) (*models.Database, error) {
return db, nil
}
// sortedConstraintNames returns constraint keys in sorted order so derived
// relationship names do not depend on map iteration order.
func sortedConstraintNames(constraints map[string]*models.Constraint) []string {
names := make([]string, 0, len(constraints))
for name := range constraints {
names = append(names, name)
}
sort.Strings(names)
return names
}
// addFKRelationship derives the relationship for a foreign key, matching how
// the PostgreSQL reader models FKs.
func addFKRelationship(table *models.Table, constraint *models.Constraint) {
name := fmt.Sprintf("%s_to_%s", table.Name, constraint.ReferencedTable)
if existing, taken := table.Relationships[name]; taken && existing.ForeignKey != constraint.Name {
// A second FK to the same table must not overwrite the first.
name = fmt.Sprintf("%s_%s", name, strings.Join(constraint.Columns, "_"))
}
relationship := models.InitRelationship(name, models.OneToMany)
relationship.FromTable = table.Name
relationship.FromSchema = table.Schema
relationship.FromColumns = append([]string(nil), constraint.Columns...)
relationship.ToTable = constraint.ReferencedTable
relationship.ToSchema = constraint.ReferencedSchema
relationship.ToColumns = append([]string(nil), constraint.ReferencedColumns...)
relationship.ForeignKey = constraint.Name
if table.Relationships == nil {
table.Relationships = make(map[string]*models.Relationship)
}
table.Relationships[name] = relationship
}
// setTableNote preserves multiple table notes. The first maps to Description
// and the second to Comment, matching the model fields used by code writers.
func setTableNote(table *models.Table, note string) {
if table.Description == "" {
table.Description = note
return
}
if table.Comment == "" {
table.Comment = note
return
}
table.Comment += "\n" + note
}
// parseColumn parses a DBML column definition
func (r *Reader) parseColumn(line, tableName, schemaName string) (*models.Column, *models.Constraint) {
// Format: column_name type [attributes] // comment
@@ -618,7 +771,7 @@ func (r *Reader) parseColumn(line, tableName, schemaName string) (*models.Column
// Parse attributes in brackets
if attrs != "" {
attrList := strings.Split(attrs, ",")
attrList := splitColumnAttrs(attrs)
for _, attr := range attrList {
attr = strings.TrimSpace(attr)
@@ -651,7 +804,9 @@ func (r *Reader) parseColumn(line, tableName, schemaName string) (*models.Column
} else if strings.HasPrefix(attr, "note:") {
// Parse column note/comment
note := strings.TrimSpace(strings.TrimPrefix(attr, "note:"))
column.Comment = strings.Trim(note, "'\"")
if !applyColumnMarkerNote(column, note) {
column.Comment = strings.Trim(note, "'\"")
}
} else if strings.HasPrefix(attr, "ref:") {
// Parse inline reference
// DBML semantics depend on context:
@@ -701,7 +856,45 @@ func (r *Reader) parseColumn(line, tableName, schemaName string) (*models.Column
return column, constraint
}
func splitInlineComment(line string) (content string, inlineComment string) {
// splitColumnAttrs splits a DBML attribute list on top-level commas. Notes and
// quoted defaults may contain commas of their own, which are part of the value
// rather than attribute separators.
func splitColumnAttrs(attrs string) []string {
var result []string
start := 0
var quote byte
escaped := false
for i := 0; i < len(attrs); i++ {
ch := attrs[i]
if quote != 0 {
if escaped {
escaped = false
continue
}
if ch == '\\' {
escaped = true
continue
}
if ch == quote {
quote = 0
}
continue
}
switch ch {
case '\'', '"', '`':
quote = ch
case ',':
result = append(result, attrs[start:i])
start = i + 1
}
}
return append(result, attrs[start:])
}
func splitInlineComment(line string) (content, inlineComment string) {
commentStart := strings.Index(line, "//")
if commentStart == -1 {
return line, ""
@@ -710,7 +903,7 @@ func splitInlineComment(line string) (content string, inlineComment string) {
return strings.TrimSpace(line[:commentStart]), strings.TrimSpace(line[commentStart+2:])
}
func splitColumnSignatureAndAttrs(line string) (signature string, attrs string) {
func splitColumnSignatureAndAttrs(line string) (signature, attrs string) {
trimmed := strings.TrimSpace(line)
if trimmed == "" || !strings.HasSuffix(trimmed, "]") {
return trimmed, ""
@@ -736,7 +929,7 @@ func splitColumnSignatureAndAttrs(line string) (signature string, attrs string)
return trimmed, ""
}
func parseColumnSignature(signature string) (columnName string, columnType string, ok bool) {
func parseColumnSignature(signature string) (columnName, columnType string, ok bool) {
signature = strings.TrimSpace(signature)
if signature == "" {
return "", "", false
@@ -1041,5 +1234,5 @@ func (r *Reader) parseTableRef(ref string) (schema, table string, columns []stri
table = stripQuotes(parts[0])
}
return
return schema, table, columns
}
+91 -3
View File
@@ -652,6 +652,22 @@ func TestReadDirectory_TableMerging(t *testing.T) {
if emailCol.Type != "varchar(255)" {
t.Errorf("Expected email type 'varchar(255)', got '%s'", emailCol.Type)
}
// Merged columns must keep declaration order (file 1: id, email; file 3:
// name, created_at) via strictly increasing, non-colliding Sequence values
// so downstream writers do not fall back to alphabetical order.
order := []string{"id", "email", "name", "created_at"}
var prev uint
for i, name := range order {
col := usersTable.Columns[name]
if col.Sequence == 0 {
t.Fatalf("column %q has zero Sequence after merge", name)
}
if i > 0 && col.Sequence <= prev {
t.Errorf("column %q Sequence %d not greater than previous %d (order not preserved across files)", name, col.Sequence, prev)
}
prev = col.Sequence
}
}
func TestReadDirectory_CommentedRefsLast(t *testing.T) {
@@ -689,7 +705,7 @@ func TestReadDirectory_CommentedRefsLast(t *testing.T) {
func TestReadDirectory_EmptyDirectory(t *testing.T) {
// Create a temporary empty directory
tmpDir := filepath.Join("..", "..", "..", "tests", "assets", "dbml", "empty_test_dir")
err := os.MkdirAll(tmpDir, 0755)
err := os.MkdirAll(tmpDir, 0o755)
if err != nil {
t.Fatalf("Failed to create temp directory: %v", err)
}
@@ -956,7 +972,7 @@ func TestReader_CompositePKIndex(t *testing.T) {
`
dir := t.TempDir()
path := filepath.Join(dir, "composite_pk.dbml")
if err := os.WriteFile(path, []byte(dbmlContent), 0644); err != nil {
if err := os.WriteFile(path, []byte(dbmlContent), 0o644); err != nil {
t.Fatalf("failed to write fixture: %v", err)
}
@@ -1005,7 +1021,7 @@ func TestReader_ColumnPKOrderPreserved(t *testing.T) {
`
dir := t.TempDir()
path := filepath.Join(dir, "column_pk_order.dbml")
if err := os.WriteFile(path, []byte(dbmlContent), 0644); err != nil {
if err := os.WriteFile(path, []byte(dbmlContent), 0o644); err != nil {
t.Fatalf("failed to write fixture: %v", err)
}
@@ -1033,3 +1049,75 @@ func TestReader_ColumnPKOrderPreserved(t *testing.T) {
t.Errorf("expected snapshot_id (declared first) to have a lower Sequence than artifact_id, got %d >= %d", snapshotCol.Sequence, artifactCol.Sequence)
}
}
func TestReader_MultilineTableNote(t *testing.T) {
dbmlContent := "Table \"info\".\"city\" {\n" +
" \"id_city\" serial [pk, not null, increment]\n" +
" \"name\" text [not null, note: 'first, second, third']\n\n" +
" note: '''\n" +
" Cities and municipalities worldwide.\n\n" +
" SPATIAL:\n" +
" Proximity queries use a GiST index.\n" +
" '''\n" +
" Note: 'Short summary'\n" +
"}\n"
path := filepath.Join(t.TempDir(), "city.dbml")
if err := os.WriteFile(path, []byte(dbmlContent), 0o644); err != nil {
t.Fatalf("failed to write fixture: %v", err)
}
db, err := NewReader(&readers.ReaderOptions{FilePath: path}).ReadDatabase()
if err != nil {
t.Fatalf("ReadDatabase() error = %v", err)
}
table := db.Schemas[0].Tables[0]
if got, want := table.Description, "Cities and municipalities worldwide.\n\nSPATIAL:\nProximity queries use a GiST index."; got != want {
t.Errorf("table description = %q, want %q", got, want)
}
if got, want := table.Comment, "Short summary"; got != want {
t.Errorf("table comment = %q, want %q", got, want)
}
if got, want := len(table.Columns), 2; got != want {
t.Errorf("column count = %d, want %d; note body must not be parsed as columns", got, want)
}
if got, want := table.Columns["name"].Comment, "first, second, third"; got != want {
t.Errorf("column note = %q, want %q", got, want)
}
}
// A name reused inside one Indexes block must not silently drop the earlier
// index; both are kept so the inspector can report the duplicate.
func TestReader_DuplicateIndexNameInTableKept(t *testing.T) {
dbmlContent := `Table individual_actor {
id bigint [pk]
rid_actor bigint
kind text
Indexes {
(rid_actor) [name: 'idx_actor', unique]
(rid_actor, kind) [name: 'idx_actor']
}
}
`
dir := t.TempDir()
path := filepath.Join(dir, "dup_index.dbml")
if err := os.WriteFile(path, []byte(dbmlContent), 0o644); err != nil {
t.Fatalf("failed to write fixture: %v", err)
}
db, err := NewReader(&readers.ReaderOptions{FilePath: path}).ReadDatabase()
if err != nil {
t.Fatalf("ReadDatabase() error = %v", err)
}
table := db.Schemas[0].Tables[0]
if len(table.Indexes) != 2 {
t.Fatalf("expected 2 indexes, got %d: %v", len(table.Indexes), table.Indexes)
}
for key, idx := range table.Indexes {
if idx.Name != "idx_actor" {
t.Errorf("index %q: Name = %q, want idx_actor", key, idx.Name)
}
}
}
+1
View File
@@ -85,6 +85,7 @@ export const postsRelations = relations(posts, ({ one }) => ({
## Notes
- `.generatedAlwaysAs(sql`expr`)` sets `Generated` + `GenerationExpression`; `.generatedAlwaysAsIdentity()` / `.generatedByDefaultAsIdentity()` set `Identity` + `IdentityGeneration`
- Supports both PostgreSQL and MySQL Drizzle schemas
- Extracts relationship information from `relations` definitions
- Schema defaults to `public` for PostgreSQL
+47 -1
View File
@@ -495,9 +495,24 @@ func (r *Reader) parseColumnModifiers(line string, column *models.Column, table
}
}
// Check for .generatedAlwaysAsIdentity()
// Check for .generatedAlwaysAsIdentity() / .generatedByDefaultAsIdentity()
if strings.Contains(line, ".generatedAlwaysAsIdentity()") {
column.AutoIncrement = true
column.Identity = true
column.IdentityGeneration = "ALWAYS"
}
if strings.Contains(line, ".generatedByDefaultAsIdentity()") {
column.AutoIncrement = true
column.Identity = true
column.IdentityGeneration = "BY DEFAULT"
}
// Check for .generatedAlwaysAs(sql`expr`) (generated column)
if idx := strings.Index(line, ".generatedAlwaysAs("); idx != -1 {
if expr, ok := parseGeneratedAlwaysAs(line[idx+len(".generatedAlwaysAs("):]); ok {
column.Generated = true
column.GenerationExpression = expr
}
}
// Check for .references(() => otherTable.column)
@@ -615,3 +630,34 @@ func (r *Reader) varNameToTableName(varName string) string {
// For now, assume variable name matches table name
return varName
}
// parseGeneratedAlwaysAs extracts the SQL expression from the argument list of
// generatedAlwaysAs(...), which follows the opening parenthesis in rest. The expression is
// the contents of a sql`...` template (backticks inside it are escaped with a backslash)
// or, failing that, a quoted string.
func parseGeneratedAlwaysAs(rest string) (string, bool) {
rest = strings.TrimSpace(rest)
rest = strings.TrimPrefix(rest, "sql")
if rest == "" {
return "", false
}
quote := rest[0]
if quote != '`' && quote != '\'' && quote != '"' {
return "", false
}
var sb strings.Builder
for i := 1; i < len(rest); i++ {
ch := rest[i]
if ch == '\\' && i+1 < len(rest) {
i++
sb.WriteByte(rest[i])
continue
}
if ch == quote {
return strings.TrimSpace(sb.String()), true
}
sb.WriteByte(ch)
}
return "", false
}
+2
View File
@@ -78,6 +78,8 @@ The reader recognizes the following GORM struct tags:
- `not null` - NOT NULL constraint
- `autoIncrement` - Auto-increment column
- `default` - Default value
- `generated` - Generated column marker (`Generated`; expression is not in the tag)
- `identity` - Identity column marker (`Identity`, `ALWAYS`)
- `size` - Column size/length
- `index` - Create index
- `uniqueIndex` - Create unique index
+48
View File
@@ -0,0 +1,48 @@
package gorm
import (
"os"
"path/filepath"
"testing"
"git.warky.dev/wdevs/relspecgo/pkg/readers"
)
func TestReader_GeneratedAndIdentityMarkers(t *testing.T) {
src := `package models
type Person struct {
ID int64 ` + "`gorm:\"column:id;primaryKey;autoIncrement;type:bigint;identity\"`" + `
Seq int64 ` + "`gorm:\"column:seq;type:bigint;<-:false;identity;not null\"`" + `
FullName string ` + "`gorm:\"column:full_name;type:text;<-:false;generated\"`" + `
First string ` + "`gorm:\"column:first;type:text\"`" + `
}
func (Person) TableName() string {
return "people"
}
`
path := filepath.Join(t.TempDir(), "person.go")
if err := os.WriteFile(path, []byte(src), 0o644); err != nil {
t.Fatal(err)
}
db, err := NewReader(&readers.ReaderOptions{FilePath: path}).ReadDatabase()
if err != nil {
t.Fatalf("ReadDatabase() error = %v", err)
}
cols := db.Schemas[0].Tables[0].Columns
if !cols["id"].Identity || cols["id"].IdentityGeneration != "ALWAYS" {
t.Errorf("id: identity = %v/%q, want true/ALWAYS", cols["id"].Identity, cols["id"].IdentityGeneration)
}
if !cols["seq"].Identity {
t.Error("seq should be identity")
}
if !cols["full_name"].Generated {
t.Error("full_name should be generated")
}
if cols["first"].Generated || cols["first"].Identity {
t.Error("first must be an ordinary column")
}
}
+15 -6
View File
@@ -246,7 +246,7 @@ func (r *Reader) getReceiverType(expr ast.Expr) string {
}
// parseTableNameMethod parses a TableName() method and extracts the table and schema name
func (r *Reader) parseTableNameMethod(funcDecl *ast.FuncDecl) (tableName string, schemaName string) {
func (r *Reader) parseTableNameMethod(funcDecl *ast.FuncDecl) (tableName, schemaName string) {
if funcDecl.Body == nil {
return "", ""
}
@@ -669,7 +669,7 @@ func (r *Reader) parseIndexesFromTag(table *models.Table, column *models.Column,
}
// extractTableFromGormTag extracts table and schema from gorm tag
func (r *Reader) extractTableFromGormTag(tag string) (tablename string, schemaName string) {
func (r *Reader) extractTableFromGormTag(tag string) (tablename, schemaName string) {
// This is typically set via TableName() method, not in tags
// We'll return empty strings and rely on deriveTableName
return "", ""
@@ -726,6 +726,15 @@ func (r *Reader) parseColumn(fieldName string, fieldType ast.Expr, tag string, s
if _, ok := parts["autoincrement"]; ok {
column.AutoIncrement = true
}
if _, ok := parts["generated"]; ok {
// GENERATED ... STORED marker written by the GORM writer; the expression is not in the tag
column.Generated = true
}
if _, ok := parts["identity"]; ok {
// GENERATED ALWAYS AS IDENTITY marker written by the GORM writer
column.Identity = true
column.IdentityGeneration = "ALWAYS"
}
if def, ok := parts["default"]; ok {
// Default value from GORM tag (e.g., default:gen_random_uuid())
column.Default = def
@@ -794,12 +803,12 @@ func (r *Reader) parseTypeWithLength(typeStr string) (baseType string, length in
if pgsql.SupportsLength(rawBaseType) && !strings.Contains(parens, ",") {
if _, err := fmt.Sscanf(parens, "%d", &length); err == nil {
baseType = pgsql.CanonicalizeBaseType(rawBaseType)
return
return baseType, length
}
}
}
return
return baseType, length
}
// parseTypeWithReferences parses a type string and extracts base type, length, and references
@@ -816,12 +825,12 @@ func (r *Reader) parseTypeWithReferences(typeStr string) (baseType string, lengt
// Parse base type for length
baseType, length = r.parseTypeWithLength(baseTypePart)
return
return baseType, length, refInfo
}
// No references, just parse type and length
baseType, length = r.parseTypeWithLength(typeStr)
return
return baseType, length, refInfo
}
// parseGormTag parses a gorm tag string into a map
+1 -1
View File
@@ -32,7 +32,7 @@ func (r *Reader) isScalarType(typeName string, ctx *parseContext) bool {
return commonCustomScalars[typeName]
}
func (r *Reader) graphQLTypeToSQL(gqlType string, fieldName string, typeName string) string {
func (r *Reader) graphQLTypeToSQL(gqlType, fieldName, typeName string) string {
// Check for ID type with configurable mapping
if gqlType == "ID" {
// Check metadata for ID type preference
+5
View File
@@ -25,6 +25,11 @@ sqlserver://user:pass@192.168.1.100:1433/production
sqlserver://localhost/testdb?encrypt=disable
```
## Computed and Identity Columns
- `sys.computed_columns` -> `Generated` + `GenerationExpression` (outer parentheses removed; persisted or not)
- Identity columns -> `Identity` + `IdentityGeneration = ALWAYS`
## Supported Constraints
- Primary Keys
+37 -2
View File
@@ -104,8 +104,12 @@ func (r *Reader) queryColumns(schemaName string) (map[string]map[string]*models.
c.numeric_precision,
c.numeric_scale,
ISNULL(ep.value, '') as description,
COLUMNPROPERTY(OBJECT_ID(QUOTENAME(c.table_schema) + '.' + QUOTENAME(c.table_name)), c.column_name, 'IsIdentity') as is_identity
COLUMNPROPERTY(OBJECT_ID(QUOTENAME(c.table_schema) + '.' + QUOTENAME(c.table_name)), c.column_name, 'IsIdentity') as is_identity,
cc.definition as computed_definition
FROM information_schema.columns c
LEFT JOIN sys.computed_columns cc
ON cc.object_id = OBJECT_ID(QUOTENAME(c.table_schema) + '.' + QUOTENAME(c.table_name))
AND cc.name = c.column_name
LEFT JOIN sys.extended_properties ep
ON ep.major_id = OBJECT_ID(QUOTENAME(c.table_schema) + '.' + QUOTENAME(c.table_name))
AND ep.minor_id = COLUMNPROPERTY(OBJECT_ID(QUOTENAME(c.table_schema) + '.' + QUOTENAME(c.table_name)), c.column_name, 'ColumnId')
@@ -127,8 +131,9 @@ func (r *Reader) queryColumns(schemaName string) (map[string]map[string]*models.
var schema, tableName, columnName, isNullable, dataType, description string
var ordinalPosition int
var columnDefault, charMaxLength, numPrecision, numScale, isIdentity *int
var computedDefinition *string
if err := rows.Scan(&schema, &tableName, &columnName, &ordinalPosition, &columnDefault, &isNullable, &dataType, &charMaxLength, &numPrecision, &numScale, &description, &isIdentity); err != nil {
if err := rows.Scan(&schema, &tableName, &columnName, &ordinalPosition, &columnDefault, &isNullable, &dataType, &charMaxLength, &numPrecision, &numScale, &description, &isIdentity, &computedDefinition); err != nil {
return nil, err
}
@@ -144,6 +149,14 @@ func (r *Reader) queryColumns(schemaName string) (map[string]map[string]*models.
// Check if this is an identity column (auto-increment)
if isIdentity != nil && *isIdentity == 1 {
column.AutoIncrement = true
column.Identity = true
column.IdentityGeneration = "ALWAYS"
}
// Computed columns report their expression wrapped in an extra pair of parentheses
if computedDefinition != nil && strings.TrimSpace(*computedDefinition) != "" {
column.Generated = true
column.GenerationExpression = trimOuterParens(*computedDefinition)
}
if charMaxLength != nil && *charMaxLength > 0 {
@@ -414,3 +427,25 @@ func (r *Reader) queryIndexes(schemaName string) (map[string][]*models.Index, er
return indexes, rows.Err()
}
// trimOuterParens removes one pair of parentheses wrapping the whole expression, as
// SQL Server stores computed column definitions.
func trimOuterParens(expr string) string {
expr = strings.TrimSpace(expr)
if len(expr) < 2 || expr[0] != '(' || expr[len(expr)-1] != ')' {
return expr
}
depth := 0
for i := 0; i < len(expr); i++ {
switch expr[i] {
case '(':
depth++
case ')':
depth--
if depth == 0 && i != len(expr)-1 {
return expr
}
}
}
return strings.TrimSpace(expr[1 : len(expr)-1])
}
+23 -7
View File
@@ -3,9 +3,10 @@ package mssql
import (
"testing"
"github.com/stretchr/testify/assert"
"git.warky.dev/wdevs/relspecgo/pkg/mssql"
"git.warky.dev/wdevs/relspecgo/pkg/readers"
"github.com/stretchr/testify/assert"
)
// TestMapDataType tests MSSQL type mapping to canonical types
@@ -38,9 +39,9 @@ func TestMapDataType(t *testing.T) {
// TestConvertCanonicalToMSSQL tests canonical to MSSQL type conversion
func TestConvertCanonicalToMSSQL(t *testing.T) {
tests := []struct {
name string
canonicalType string
expectedMSSQL string
name string
canonicalType string
expectedMSSQL string
}{
{"int to INT", "int", "INT"},
{"int64 to BIGINT", "int64", "BIGINT"},
@@ -63,9 +64,9 @@ func TestConvertCanonicalToMSSQL(t *testing.T) {
// TestConvertMSSQLToCanonical tests MSSQL to canonical type conversion
func TestConvertMSSQLToCanonical(t *testing.T) {
tests := []struct {
name string
mssqlType string
expectedType string
name string
mssqlType string
expectedType string
}{
{"INT to int", "INT", "int"},
{"BIGINT to int64", "BIGINT", "int64"},
@@ -84,3 +85,18 @@ func TestConvertMSSQLToCanonical(t *testing.T) {
})
}
}
func TestTrimOuterParens(t *testing.T) {
tests := map[string]string{
"(([a])+([b]))": "([a])+([b])",
"([a]+[b])": "[a]+[b]",
"([a])+([b])": "([a])+([b])",
"[a]+[b]": "[a]+[b]",
" (upper([a])) ": "upper([a])",
}
for in, want := range tests {
if got := trimOuterParens(in); got != want {
t.Errorf("trimOuterParens(%q) = %q, want %q", in, got, want)
}
}
}
+72 -27
View File
@@ -243,7 +243,11 @@ func (r *Reader) queryColumns(schemaName string) (map[string]map[string]*models.
c.numeric_scale,
c.udt_name,
pg_catalog.format_type(a.atttypid, a.atttypmod) as formatted_data_type,
col_description((c.table_schema||'.'||c.table_name)::regclass, c.ordinal_position) as description
col_description((c.table_schema||'.'||c.table_name)::regclass, c.ordinal_position) as description,
c.is_generated,
c.generation_expression,
c.is_identity,
c.identity_generation
FROM information_schema.columns c
JOIN pg_catalog.pg_namespace n
ON n.nspname = c.table_schema
@@ -268,17 +272,36 @@ func (r *Reader) queryColumns(schemaName string) (map[string]map[string]*models.
columnsMap := make(map[string]map[string]*models.Column)
for rows.Next() {
var schema, tableName, columnName, isNullable, dataType, udtName, formattedDataType string
var schema, tableName, columnName, isNullable, dataType, udtName, formattedDataType, isGenerated, isIdentity string
var ordinalPosition int
var columnDefault, description *string
var columnDefault, description, generationExpression, identityGeneration *string
var charMaxLength, numPrecision, numScale *int
if err := rows.Scan(&schema, &tableName, &columnName, &ordinalPosition, &columnDefault, &isNullable, &dataType, &charMaxLength, &numPrecision, &numScale, &udtName, &formattedDataType, &description); err != nil {
if err := rows.Scan(&schema, &tableName, &columnName, &ordinalPosition, &columnDefault, &isNullable, &dataType, &charMaxLength, &numPrecision, &numScale, &udtName, &formattedDataType, &description, &isGenerated, &generationExpression, &isIdentity, &identityGeneration); err != nil {
return nil, err
}
column := models.InitColumn(columnName, tableName, schema)
// GENERATED ALWAYS ... STORED columns are computed by Postgres and cannot
// have their default altered/dropped like a regular column.
column.Generated = isGenerated == "ALWAYS"
if generationExpression != nil {
column.GenerationExpression = *generationExpression
}
// GENERATED { ALWAYS | BY DEFAULT } AS IDENTITY columns are driven by an
// internal sequence rather than a literal default (unlike serial columns,
// they carry no pg_attrdef row at all), so they need the same DB-side
// handling as generated columns even though the underlying mechanism differs.
column.Identity = isIdentity == "YES"
if identityGeneration != nil {
column.IdentityGeneration = strings.ToUpper(strings.TrimSpace(*identityGeneration))
}
if column.Identity {
column.AutoIncrement = true
}
// Check if this is a serial type (has nextval default)
hasNextval := false
if columnDefault != nil {
@@ -301,6 +324,24 @@ func (r *Reader) queryColumns(schemaName string) (map[string]map[string]*models.
column.Description = *description
}
if column.Generated {
note := "GENERATED ALWAYS AS (" + column.GenerationExpression + ") STORED"
if column.Description != "" {
column.Description = column.Description + " " + note
} else {
column.Description = note
}
}
if column.Identity {
note := "GENERATED " + column.IdentityGeneration + " AS IDENTITY"
if column.Description != "" {
column.Description = column.Description + " " + note
} else {
column.Description = note
}
}
if charMaxLength != nil {
column.Length = *charMaxLength
}
@@ -381,31 +422,35 @@ func (r *Reader) queryPrimaryKeys(schemaName string) (map[string]*models.Constra
// queryForeignKeys retrieves all foreign key constraints for a schema
// Returns map[schema.table][]*Constraint
func (r *Reader) queryForeignKeys(schemaName string) (map[string][]*models.Constraint, error) {
// Columns are paired by position from pg_constraint so composite keys are not
// cross-joined (information_schema.constraint_column_usage has no ordering).
actionCase := func(col string) string {
return `CASE ` + col + ` WHEN 'a' THEN 'NO ACTION' WHEN 'r' THEN 'RESTRICT' WHEN 'c' THEN 'CASCADE' WHEN 'n' THEN 'SET NULL' WHEN 'd' THEN 'SET DEFAULT' END`
}
query := `
SELECT
tc.table_schema,
tc.table_name,
tc.constraint_name,
kcu.table_schema as foreign_table_schema,
kcu.table_name as foreign_table_name,
kcu.column_name as foreign_column,
ccu.table_schema as referenced_table_schema,
ccu.table_name as referenced_table_name,
ccu.column_name as referenced_column,
rc.update_rule,
rc.delete_rule
FROM information_schema.table_constraints tc
JOIN information_schema.key_column_usage kcu
ON tc.constraint_name = kcu.constraint_name
AND tc.table_schema = kcu.table_schema
JOIN information_schema.constraint_column_usage ccu
ON ccu.constraint_name = tc.constraint_name
JOIN information_schema.referential_constraints rc
ON rc.constraint_name = tc.constraint_name
AND rc.constraint_schema = tc.table_schema
WHERE tc.constraint_type = 'FOREIGN KEY'
AND tc.table_schema = $1
ORDER BY tc.table_schema, tc.table_name, tc.constraint_name, kcu.ordinal_position
ns.nspname AS table_schema,
cl.relname AS table_name,
con.conname AS constraint_name,
ns.nspname AS foreign_table_schema,
cl.relname AS foreign_table_name,
att.attname AS foreign_column,
fns.nspname AS referenced_table_schema,
fcl.relname AS referenced_table_name,
fatt.attname AS referenced_column,
` + actionCase("con.confupdtype") + ` AS update_rule,
` + actionCase("con.confdeltype") + ` AS delete_rule
FROM pg_catalog.pg_constraint con
JOIN pg_catalog.pg_class cl ON cl.oid = con.conrelid
JOIN pg_catalog.pg_namespace ns ON ns.oid = cl.relnamespace
JOIN pg_catalog.pg_class fcl ON fcl.oid = con.confrelid
JOIN pg_catalog.pg_namespace fns ON fns.oid = fcl.relnamespace
CROSS JOIN LATERAL unnest(con.conkey, con.confkey) WITH ORDINALITY AS k(attnum, fattnum, ord)
JOIN pg_catalog.pg_attribute att ON att.attrelid = con.conrelid AND att.attnum = k.attnum
JOIN pg_catalog.pg_attribute fatt ON fatt.attrelid = con.confrelid AND fatt.attnum = k.fattnum
WHERE con.contype = 'f'
AND ns.nspname = $1
ORDER BY ns.nspname, cl.relname, con.conname, k.ord
`
rows, err := r.conn.Query(r.ctx, query, schemaName)
+57 -2
View File
@@ -34,11 +34,14 @@ func (r *Reader) ReadDatabase() (*models.Database, error) {
return nil, fmt.Errorf("connection string is required")
}
// Connect to the database
// Connect to the database. This can take noticeable time across a slow network,
// so report it before the driver starts the connection attempt.
r.progress("Connecting to PostgreSQL...")
if err := r.connect(); err != nil {
return nil, fmt.Errorf("failed to connect: %w", err)
}
defer r.close()
r.progress("Connected. Reading database metadata...")
// Get database name from connection
var dbName string
@@ -60,34 +63,42 @@ func (r *Reader) ReadDatabase() (*models.Database, error) {
}
// Query all schemas
r.progress("Discovering schemas...")
schemas, err := r.querySchemas()
if err != nil {
return nil, fmt.Errorf("failed to query schemas: %w", err)
}
// Process each schema
for _, schema := range schemas {
for schemaIndex, schema := range schemas {
r.progress(fmt.Sprintf("Reading schema %q (%d/%d): tables...", schema.Name, schemaIndex+1, len(schemas)))
// Query tables for this schema
tables, err := r.queryTables(schema.Name)
if err != nil {
return nil, fmt.Errorf("failed to query tables for schema %s: %w", schema.Name, err)
}
schema.Tables = tables
r.progress(fmt.Sprintf("Reading schema %q: found %d table(s).", schema.Name, len(tables)))
r.progress(fmt.Sprintf("Reading schema %q: views...", schema.Name))
// Query views for this schema
views, err := r.queryViews(schema.Name)
if err != nil {
return nil, fmt.Errorf("failed to query views for schema %s: %w", schema.Name, err)
}
schema.Views = views
r.progress(fmt.Sprintf("Reading schema %q: found %d view(s).", schema.Name, len(views)))
r.progress(fmt.Sprintf("Reading schema %q: sequences...", schema.Name))
// Query sequences for this schema
sequences, err := r.querySequences(schema.Name)
if err != nil {
return nil, fmt.Errorf("failed to query sequences for schema %s: %w", schema.Name, err)
}
schema.Sequences = sequences
r.progress(fmt.Sprintf("Reading schema %q: found %d sequence(s).", schema.Name, len(sequences)))
r.progress(fmt.Sprintf("Reading schema %q: extensions...", schema.Name))
// Query extensions installed into this schema
extensions, err := r.queryExtensions(schema.Name)
if err != nil {
@@ -99,12 +110,15 @@ func (r *Reader) ReadDatabase() (*models.Database, error) {
}
schema.Metadata["extensions"] = extensions
}
r.progress(fmt.Sprintf("Reading schema %q: found %d extension(s).", schema.Name, len(extensions)))
r.progress(fmt.Sprintf("Reading schema %q: columns...", schema.Name))
// Query columns for tables and views
columnsMap, err := r.queryColumns(schema.Name)
if err != nil {
return nil, fmt.Errorf("failed to query columns for schema %s: %w", schema.Name, err)
}
r.progress(fmt.Sprintf("Reading schema %q: found %d column(s).", schema.Name, countColumns(columnsMap)))
// Populate table columns
for _, table := range schema.Tables {
@@ -122,11 +136,13 @@ func (r *Reader) ReadDatabase() (*models.Database, error) {
}
}
r.progress(fmt.Sprintf("Reading schema %q: primary keys...", schema.Name))
// Query primary keys
primaryKeys, err := r.queryPrimaryKeys(schema.Name)
if err != nil {
return nil, fmt.Errorf("failed to query primary keys for schema %s: %w", schema.Name, err)
}
r.progress(fmt.Sprintf("Reading schema %q: found %d primary key(s).", schema.Name, len(primaryKeys)))
// Apply primary keys to tables
for _, table := range schema.Tables {
@@ -143,11 +159,13 @@ func (r *Reader) ReadDatabase() (*models.Database, error) {
}
}
r.progress(fmt.Sprintf("Reading schema %q: foreign keys...", schema.Name))
// Query foreign keys
foreignKeys, err := r.queryForeignKeys(schema.Name)
if err != nil {
return nil, fmt.Errorf("failed to query foreign keys for schema %s: %w", schema.Name, err)
}
r.progress(fmt.Sprintf("Reading schema %q: found %d foreign key(s).", schema.Name, countConstraints(foreignKeys)))
// Apply foreign keys to tables
for _, table := range schema.Tables {
@@ -161,11 +179,13 @@ func (r *Reader) ReadDatabase() (*models.Database, error) {
}
}
r.progress(fmt.Sprintf("Reading schema %q: unique constraints...", schema.Name))
// Query unique constraints
uniqueConstraints, err := r.queryUniqueConstraints(schema.Name)
if err != nil {
return nil, fmt.Errorf("failed to query unique constraints for schema %s: %w", schema.Name, err)
}
r.progress(fmt.Sprintf("Reading schema %q: found %d unique constraint(s).", schema.Name, countConstraints(uniqueConstraints)))
// Apply unique constraints to tables
for _, table := range schema.Tables {
@@ -177,11 +197,13 @@ func (r *Reader) ReadDatabase() (*models.Database, error) {
}
}
r.progress(fmt.Sprintf("Reading schema %q: check constraints...", schema.Name))
// Query check constraints
checkConstraints, err := r.queryCheckConstraints(schema.Name)
if err != nil {
return nil, fmt.Errorf("failed to query check constraints for schema %s: %w", schema.Name, err)
}
r.progress(fmt.Sprintf("Reading schema %q: found %d check constraint(s).", schema.Name, countConstraints(checkConstraints)))
// Apply check constraints to tables
for _, table := range schema.Tables {
@@ -193,11 +215,13 @@ func (r *Reader) ReadDatabase() (*models.Database, error) {
}
}
r.progress(fmt.Sprintf("Reading schema %q: indexes...", schema.Name))
// Query indexes
indexes, err := r.queryIndexes(schema.Name)
if err != nil {
return nil, fmt.Errorf("failed to query indexes for schema %s: %w", schema.Name, err)
}
r.progress(fmt.Sprintf("Reading schema %q: found %d index(es).", schema.Name, countIndexes(indexes)))
// Apply indexes to tables
for _, table := range schema.Tables {
@@ -226,10 +250,41 @@ func (r *Reader) ReadDatabase() (*models.Database, error) {
// Add schema to database
db.Schemas = append(db.Schemas, schema)
}
r.progress("PostgreSQL schema read complete.")
return db, nil
}
func (r *Reader) progress(message string) {
if r.options.Progress != nil {
r.options.Progress(message)
}
}
func countColumns(columns map[string]map[string]*models.Column) int {
total := 0
for _, tableColumns := range columns {
total += len(tableColumns)
}
return total
}
func countConstraints(constraints map[string][]*models.Constraint) int {
total := 0
for _, tableConstraints := range constraints {
total += len(tableConstraints)
}
return total
}
func countIndexes(indexes map[string][]*models.Index) int {
total := 0
for _, tableIndexes := range indexes {
total += len(tableIndexes)
}
return total
}
// ReadSchema reads a single schema (returns the first schema from the database)
func (r *Reader) ReadSchema() (*models.Schema, error) {
db, err := r.ReadDatabase()
+61
View File
@@ -1,10 +1,14 @@
package pgsql
import (
"context"
"os"
"reflect"
"strings"
"testing"
"github.com/jackc/pgx/v5"
"git.warky.dev/wdevs/relspecgo/pkg/models"
"git.warky.dev/wdevs/relspecgo/pkg/readers"
)
@@ -499,3 +503,60 @@ func TestMapDataType_ExtensionTypesPreserveModifiers(t *testing.T) {
})
}
}
func TestReader_CompositeForeignKeyColumnsArePairedOnce(t *testing.T) {
connStr := getTestConnectionString(t)
ctx := context.Background()
conn, err := pgx.Connect(ctx, connStr)
if err != nil {
t.Fatalf("connect: %v", err)
}
defer conn.Close(ctx)
const schema = "relspec_fk_test"
setup := []string{
"DROP SCHEMA IF EXISTS " + schema + " CASCADE",
"CREATE SCHEMA " + schema,
"CREATE TABLE " + schema + ".parent (a int, b int, PRIMARY KEY (a, b))",
"CREATE TABLE " + schema + ".child (x int, y int, CONSTRAINT fk_child_parent FOREIGN KEY (x, y) REFERENCES " + schema + ".parent (a, b) ON DELETE CASCADE)",
}
for _, stmt := range setup {
if _, err := conn.Exec(ctx, stmt); err != nil {
t.Fatalf("setup %q: %v", stmt, err)
}
}
defer conn.Exec(ctx, "DROP SCHEMA IF EXISTS "+schema+" CASCADE")
reader := NewReader(&readers.ReaderOptions{ConnectionString: connStr})
db, err := reader.ReadDatabase()
if err != nil {
t.Fatalf("ReadDatabase: %v", err)
}
for _, s := range db.Schemas {
if s.Name != schema {
continue
}
for _, tbl := range s.Tables {
if tbl.Name != "child" {
continue
}
fk := tbl.Constraints["fk_child_parent"]
if fk == nil {
t.Fatal("foreign key fk_child_parent not read")
}
if got := strings.Join(fk.Columns, ","); got != "x,y" {
t.Errorf("columns = %q, want x,y", got)
}
if got := strings.Join(fk.ReferencedColumns, ","); got != "a,b" {
t.Errorf("referenced columns = %q, want a,b", got)
}
if fk.OnDelete != "CASCADE" || fk.OnUpdate != "NO ACTION" {
t.Errorf("rules = %s/%s, want CASCADE/NO ACTION", fk.OnDelete, fk.OnUpdate)
}
return
}
}
t.Fatal("test schema/table not found in read result")
}
+2 -2
View File
@@ -28,7 +28,7 @@ model User {
id Int @id @default(autoincrement())
}`
if err := os.WriteFile(schemaPath, []byte(content), 0644); err != nil {
if err := os.WriteFile(schemaPath, []byte(content), 0o644); err != nil {
t.Fatalf("failed to write schema: %v", err)
}
@@ -58,7 +58,7 @@ model User {
id Int @id @default(autoincrement())
}`
if err := os.WriteFile(schemaPath, []byte(content), 0644); err != nil {
if err := os.WriteFile(schemaPath, []byte(content), 0o644); err != nil {
t.Fatalf("failed to write schema: %v", err)
}
+8
View File
@@ -28,6 +28,14 @@ type ReaderOptions struct {
// Prisma7 enables Prisma 7-specific handling for Prisma schemas.
Prisma7 bool
// StrictDirectives makes DBML dialect directives (@postgres:, @sqlite:, …)
// fail on an unknown namespace or key instead of preserving them silently.
StrictDirectives bool
// Progress receives human-readable status updates while a reader is working.
// It is optional so library users can opt in without coupling readers to a UI.
Progress func(string)
// Additional options can be added here as needed
Metadata map[string]interface{}
}
-1
View File
@@ -175,7 +175,6 @@ func (r *Reader) readScripts() ([]*models.Script, error) {
return nil
})
if err != nil {
return nil, err
}
+10 -10
View File
@@ -30,18 +30,18 @@ func TestReader_ReadDatabase(t *testing.T) {
for filename, content := range testFiles {
filePath := filepath.Join(tempDir, filename)
if err := os.WriteFile(filePath, []byte(content), 0644); err != nil {
if err := os.WriteFile(filePath, []byte(content), 0o644); err != nil {
t.Fatalf("Failed to create test file %s: %v", filename, err)
}
}
// Create subdirectory with additional script
subDir := filepath.Join(tempDir, "migrations")
if err := os.MkdirAll(subDir, 0755); err != nil {
if err := os.MkdirAll(subDir, 0o755); err != nil {
t.Fatalf("Failed to create subdirectory: %v", err)
}
subFile := filepath.Join(subDir, "3_001_add_column.sql")
if err := os.WriteFile(subFile, []byte("ALTER TABLE users ADD COLUMN email TEXT;"), 0644); err != nil {
if err := os.WriteFile(subFile, []byte("ALTER TABLE users ADD COLUMN email TEXT;"), 0o644); err != nil {
t.Fatalf("Failed to create subdirectory file: %v", err)
}
@@ -141,7 +141,7 @@ func TestReader_ReadSchema(t *testing.T) {
// Create test SQL file
testFile := filepath.Join(tempDir, "1_001_test.sql")
if err := os.WriteFile(testFile, []byte("SELECT 1;"), 0644); err != nil {
if err := os.WriteFile(testFile, []byte("SELECT 1;"), 0o644); err != nil {
t.Fatalf("Failed to create test file: %v", err)
}
@@ -220,14 +220,14 @@ func TestReader_InvalidFilename(t *testing.T) {
for _, filename := range invalidFiles {
filePath := filepath.Join(tempDir, filename)
if err := os.WriteFile(filePath, []byte("SELECT 1;"), 0644); err != nil {
if err := os.WriteFile(filePath, []byte("SELECT 1;"), 0o644); err != nil {
t.Fatalf("Failed to create test file %s: %v", filename, err)
}
}
// Create one valid file
validFile := filepath.Join(tempDir, "1_001_valid.sql")
if err := os.WriteFile(validFile, []byte("SELECT 1;"), 0644); err != nil {
if err := os.WriteFile(validFile, []byte("SELECT 1;"), 0o644); err != nil {
t.Fatalf("Failed to create valid file: %v", err)
}
@@ -277,7 +277,7 @@ func TestReader_HyphenFormat(t *testing.T) {
for filename, content := range testFiles {
filePath := filepath.Join(tempDir, filename)
if err := os.WriteFile(filePath, []byte(content), 0644); err != nil {
if err := os.WriteFile(filePath, []byte(content), 0o644); err != nil {
t.Fatalf("Failed to create test file %s: %v", filename, err)
}
}
@@ -343,7 +343,7 @@ func TestReader_MixedFormat(t *testing.T) {
for filename, content := range testFiles {
filePath := filepath.Join(tempDir, filename)
if err := os.WriteFile(filePath, []byte(content), 0644); err != nil {
if err := os.WriteFile(filePath, []byte(content), 0o644); err != nil {
t.Fatalf("Failed to create test file %s: %v", filename, err)
}
}
@@ -386,13 +386,13 @@ func TestReader_SkipSymlinks(t *testing.T) {
// Create a real SQL file
realFile := filepath.Join(tempDir, "1_001_real_file.sql")
if err := os.WriteFile(realFile, []byte("SELECT 1;"), 0644); err != nil {
if err := os.WriteFile(realFile, []byte("SELECT 1;"), 0o644); err != nil {
t.Fatalf("Failed to create real file: %v", err)
}
// Create another file to link to
targetFile := filepath.Join(tempDir, "2_001_target.sql")
if err := os.WriteFile(targetFile, []byte("SELECT 2;"), 0644); err != nil {
if err := os.WriteFile(targetFile, []byte("SELECT 2;"), 0o644); err != nil {
t.Fatalf("Failed to create target file: %v", err)
}
+1
View File
@@ -40,6 +40,7 @@ options := &readers.ReaderOptions{
- Uses pure Go driver (modernc.org/sqlite) - no CGo required
- Supports both file path and connection string
- Auto-increment detection for INTEGER PRIMARY KEY columns
- Generated columns (virtual and stored) are read via `PRAGMA table_xinfo`; the expression is parsed from the `CREATE TABLE` SQL
- Foreign keys require `PRAGMA foreign_keys = ON` to be set
## Example Schema
+116
View File
@@ -0,0 +1,116 @@
package sqlite
import (
"regexp"
"strings"
)
var generatedAsRegex = regexp.MustCompile(`(?is)\bAS\s*\(`)
// parseGeneratedExpression extracts the expression of a generated column from a CREATE
// TABLE statement, e.g. `full TEXT GENERATED ALWAYS AS (a || b) STORED` yields `a || b`.
// It returns "" when the column or its expression cannot be found.
func parseGeneratedExpression(createSQL, columnName string) string {
open := strings.Index(createSQL, "(")
if open < 0 {
return ""
}
for _, def := range splitTopLevel(createSQL[open+1:]) {
if !strings.EqualFold(firstIdentifier(def), columnName) {
continue
}
loc := generatedAsRegex.FindStringIndex(def)
if loc == nil {
return ""
}
return balancedParens(def[loc[1]-1:])
}
return ""
}
// splitTopLevel splits a table body on commas that are outside parentheses and quotes,
// stopping at the parenthesis that closes the body.
func splitTopLevel(body string) []string {
var parts []string
depth := 0
var quote byte
start := 0
for i := 0; i < len(body); i++ {
ch := body[i]
if quote != 0 {
if ch == quote {
quote = 0
}
continue
}
switch ch {
case '\'', '"', '`':
quote = ch
case '[':
quote = ']'
case '(':
depth++
case ')':
if depth == 0 {
return append(parts, body[start:i])
}
depth--
case ',':
if depth == 0 {
parts = append(parts, body[start:i])
start = i + 1
}
}
}
return append(parts, body[start:])
}
// firstIdentifier returns the first (possibly quoted) identifier of a column definition.
func firstIdentifier(def string) string {
def = strings.TrimSpace(def)
if def == "" {
return ""
}
switch def[0] {
case '"', '\'', '`':
if end := strings.IndexByte(def[1:], def[0]); end >= 0 {
return def[1 : 1+end]
}
case '[':
if end := strings.IndexByte(def, ']'); end >= 0 {
return def[1:end]
}
}
if end := strings.IndexAny(def, " \t\r\n"); end >= 0 {
return def[:end]
}
return def
}
// balancedParens returns the text inside the parenthesis group that starts at s[0].
func balancedParens(s string) string {
depth := 0
var quote byte
for i := 0; i < len(s); i++ {
ch := s[i]
if quote != 0 {
if ch == quote {
quote = 0
}
continue
}
switch ch {
case '\'', '"':
quote = ch
case '(':
depth++
case ')':
depth--
if depth == 0 {
return strings.TrimSpace(s[1:i])
}
}
}
return ""
}
+64
View File
@@ -0,0 +1,64 @@
package sqlite
import (
"database/sql"
"path/filepath"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"git.warky.dev/wdevs/relspecgo/pkg/readers"
)
func TestParseGeneratedExpression(t *testing.T) {
createSQL := `CREATE TABLE people (
id INTEGER PRIMARY KEY,
"first" TEXT,
last TEXT,
full_name TEXT GENERATED ALWAYS AS (coalesce("first", '') || ' ' || last) STORED,
initials TEXT AS (substr("first", 1, 1) || substr(last, 1, 1)),
plain TEXT NOT NULL
)`
tests := []struct {
column string
want string
}{
{"full_name", `coalesce("first", '') || ' ' || last`},
{"initials", `substr("first", 1, 1) || substr(last, 1, 1)`},
{"plain", ""},
{"missing", ""},
}
for _, tt := range tests {
t.Run(tt.column, func(t *testing.T) {
assert.Equal(t, tt.want, parseGeneratedExpression(createSQL, tt.column))
})
}
}
func TestReader_GeneratedColumns(t *testing.T) {
dbPath := filepath.Join(t.TempDir(), "gen.db")
db, err := sql.Open("sqlite", dbPath)
require.NoError(t, err)
_, err = db.Exec(`CREATE TABLE people (
id INTEGER PRIMARY KEY,
first TEXT,
last TEXT,
full_name TEXT GENERATED ALWAYS AS (first || ' ' || last) STORED,
initials TEXT AS (substr(first, 1, 1) || substr(last, 1, 1))
)`)
require.NoError(t, err)
require.NoError(t, db.Close())
got, err := NewReader(&readers.ReaderOptions{FilePath: dbPath}).ReadDatabase()
require.NoError(t, err)
cols := got.Schemas[0].Tables[0].Columns
require.Contains(t, cols, "full_name", "generated columns must be read")
assert.True(t, cols["full_name"].Generated)
assert.Equal(t, "first || ' ' || last", cols["full_name"].GenerationExpression)
assert.True(t, cols["initials"].Generated)
assert.Equal(t, "substr(first, 1, 1) || substr(last, 1, 1)", cols["initials"].GenerationExpression)
assert.False(t, cols["first"].Generated)
}
+29 -4
View File
@@ -75,7 +75,8 @@ func (r *Reader) queryViews() ([]*models.View, error) {
// queryColumns retrieves all columns for a given table or view
func (r *Reader) queryColumns(tableName string) (map[string]*models.Column, error) {
query := fmt.Sprintf("PRAGMA table_info(%s)", tableName)
// table_xinfo, unlike table_info, also lists generated columns (hidden = 2 virtual, 3 stored)
query := fmt.Sprintf("PRAGMA table_xinfo(%s)", tableName)
rows, err := r.db.QueryContext(r.ctx, query)
if err != nil {
@@ -84,24 +85,38 @@ func (r *Reader) queryColumns(tableName string) (map[string]*models.Column, erro
defer rows.Close()
columns := make(map[string]*models.Column)
var tableSQL string
tableSQLLoaded := false
for rows.Next() {
var cid int
var name, dataType string
var notNull, pk int
var notNull, pk, hidden int
var defaultValue *string
if err := rows.Scan(&cid, &name, &dataType, &notNull, &defaultValue, &pk); err != nil {
if err := rows.Scan(&cid, &name, &dataType, &notNull, &defaultValue, &pk, &hidden); err != nil {
return nil, err
}
// Hidden virtual-table columns (hidden = 1) are not part of the schema
if hidden == 1 {
continue
}
column := models.InitColumn(name, tableName, "main")
column.Type = r.mapDataType(strings.ToUpper(dataType))
column.NotNull = (notNull == 1)
column.IsPrimaryKey = (pk > 0)
column.Sequence = uint(cid + 1)
if defaultValue != nil {
if hidden == 2 || hidden == 3 {
column.Generated = true
if !tableSQLLoaded {
tableSQL = r.tableSQL(tableName)
tableSQLLoaded = true
}
column.GenerationExpression = parseGeneratedExpression(tableSQL, name)
} else if defaultValue != nil {
column.Default = *defaultValue
}
@@ -116,6 +131,16 @@ func (r *Reader) queryColumns(tableName string) (map[string]*models.Column, erro
return columns, rows.Err()
}
// tableSQL returns the CREATE TABLE statement of a table, or "" when it cannot be read.
func (r *Reader) tableSQL(tableName string) string {
var sql string
err := r.db.QueryRowContext(r.ctx, `SELECT sql FROM sqlite_master WHERE type = 'table' AND name = ?`, tableName).Scan(&sql)
if err != nil {
return ""
}
return sql
}
// isAutoIncrement checks if a column is autoincrement
func (r *Reader) isAutoIncrement(tableName, columnName string) bool {
// Check sqlite_sequence table or parse CREATE TABLE statement
+2
View File
@@ -114,6 +114,8 @@ export class Post {
- `@JoinColumn()` - Foreign key column
- `@Index()` - Index definition
- `@Unique()` - Unique constraint
- `asExpression` / `generatedType` - Generated column (`Generated` + `GenerationExpression`)
- `@PrimaryGeneratedColumn('identity')`, `@Generated('identity')`, `generatedIdentity` - Identity column
## Notes
+94 -4
View File
@@ -128,7 +128,6 @@ func (r *Reader) extractEntities(content string) []entityInfo {
scanner := bufio.NewScanner(strings.NewReader(content))
entityRegex := regexp.MustCompile(`^export\s+class\s+(\w+)`)
decoratorRegex := regexp.MustCompile(`^\s*@(\w+)(\([^)]*\))?`)
fieldRegex := regexp.MustCompile(`^\s*(\w+):\s*([^;]+);`)
var currentEntity *entityInfo
@@ -145,8 +144,7 @@ func (r *Reader) extractEntities(content string) []entityInfo {
}
// Check for decorator
if matches := decoratorRegex.FindStringSubmatch(trimmed); matches != nil {
decorator := matches[0]
if decorator, ok := matchDecorator(trimmed); ok {
pendingDecorators = append(pendingDecorators, decorator)
continue
}
@@ -488,7 +486,11 @@ func (r *Reader) parseColumnDecorator(decorator string, column *models.Column, t
column.IsPrimaryKey = true
column.NotNull = true
if strings.Contains(decorator, "'uuid'") {
if strings.Contains(decorator, "'identity'") {
column.AutoIncrement = true
column.Identity = true
column.IdentityGeneration = parseGeneratedIdentity(decorator)
} else if strings.Contains(decorator, "'uuid'") {
column.Type = "uuid"
column.Default = "gen_random_uuid()"
} else if strings.Contains(decorator, "'increment'") || strings.Contains(decorator, "()") {
@@ -497,6 +499,17 @@ func (r *Reader) parseColumnDecorator(decorator string, column *models.Column, t
return
}
// @Generated('identity') on a non-key column; generatedIdentity is read from @Column
if strings.HasPrefix(decorator, "@Generated") {
if strings.Contains(decorator, "'identity'") {
column.Identity = true
if column.IdentityGeneration == "" {
column.IdentityGeneration = "BY DEFAULT"
}
}
return
}
// @Column
if strings.HasPrefix(decorator, "@Column") {
r.parseColumnOptions(decorator, column, table)
@@ -586,6 +599,14 @@ func (r *Reader) parseColumnOptions(decorator string, column *models.Column, tab
}
}
if matches := asExpressionRegex.FindStringSubmatch(content); matches != nil {
column.Generated = true
column.GenerationExpression = unescapeSingleQuoted(matches[1])
}
if strings.Contains(content, "generatedIdentity") {
column.IdentityGeneration = parseGeneratedIdentity(content)
}
if strings.Contains(content, "nullable: true") || strings.Contains(content, "nullable:true") {
column.NotNull = false
}
@@ -834,3 +855,72 @@ func (r *Reader) getPrimaryKeyColumn(table *models.Table) *models.Column {
return pk
}
var (
asExpressionRegex = regexp.MustCompile(`asExpression:\s*'((?:\\.|[^\\'])*)'`)
generatedIdentityRegexp = regexp.MustCompile(`generatedIdentity:\s*['"](ALWAYS|BY DEFAULT)['"]`)
)
// parseGeneratedIdentity returns the identity generation mode named in a decorator
// ("ALWAYS" or "BY DEFAULT"), defaulting to "BY DEFAULT" as TypeORM does.
func parseGeneratedIdentity(decorator string) string {
if matches := generatedIdentityRegexp.FindStringSubmatch(decorator); matches != nil {
return matches[1]
}
return "BY DEFAULT"
}
// unescapeSingleQuoted reverses escaping applied inside a quoted TypeScript string.
func unescapeSingleQuoted(s string) string {
var sb strings.Builder
for i := 0; i < len(s); i++ {
if s[i] == '\\' && i+1 < len(s) {
i++
}
sb.WriteByte(s[i])
}
return sb.String()
}
var decoratorNameRegex = regexp.MustCompile(`^@\w+`)
// matchDecorator returns the decorator at the start of line, including its argument list.
// Parentheses inside quoted strings (e.g. a generated column expression) do not end it.
func matchDecorator(line string) (string, bool) {
name := decoratorNameRegex.FindString(line)
if name == "" {
return "", false
}
rest := line[len(name):]
if !strings.HasPrefix(rest, "(") {
return name, true
}
depth := 0
var quote byte
for i := 0; i < len(rest); i++ {
ch := rest[i]
if quote != 0 {
switch ch {
case '\\':
i++
case quote:
quote = 0
}
continue
}
switch ch {
case '\'', '"', '`':
quote = ch
case '(':
depth++
case ')':
depth--
if depth == 0 {
return name + rest[:i+1], true
}
}
}
// Unterminated argument list: keep the whole line, as a best effort
return line, true
}
+1 -1
View File
@@ -192,7 +192,7 @@ func mapKeyLess(a, b reflect.Value) bool {
// MapGet safely gets a value from a map by key
// Returns nil if key doesn't exist or not a map
func MapGet(m interface{}, key interface{}) interface{} {
func MapGet(m, key interface{}) interface{} {
v := reflect.ValueOf(m)
v, ok := Deref(v)
if !ok {
+13 -12
View File
@@ -111,6 +111,7 @@ func (n *SqlNull[T]) Scan(value any) error {
return n.FromString(fmt.Sprintf("%v", value))
}
}
func (n *SqlNull[T]) FromString(s string) error {
s = strings.TrimSpace(s)
n.Valid = false
@@ -444,55 +445,55 @@ type (
SqlUUID = SqlNull[uuid.UUID]
)
// SqlTimeStamp - Timestamp with custom formatting (YYYY-MM-DDTHH:MM:SS).
// SqlTimeStamp - Timestamp serialized as RFC3339.
type SqlTimeStamp struct{ SqlNull[time.Time] }
func (t SqlTimeStamp) MarshalJSON() ([]byte, error) {
if !t.Valid || t.Val.IsZero() || t.Val.Before(time.Date(0002, 1, 1, 0, 0, 0, 0, time.UTC)) {
if !t.Valid || t.Val.IsZero() || t.Val.Before(time.Date(0o002, 1, 1, 0, 0, 0, 0, time.UTC)) {
return []byte("null"), nil
}
return fmt.Appendf(nil, `"%s"`, t.Val.Format("2006-01-02T15:04:05")), nil
return fmt.Appendf(nil, `"%s"`, t.Val.Format(time.RFC3339)), nil
}
func (t *SqlTimeStamp) UnmarshalJSON(b []byte) error {
if err := t.SqlNull.UnmarshalJSON(b); err != nil {
return err
}
if t.Valid && (t.Val.IsZero() || t.Val.Format("2006-01-02T15:04:05") == "0001-01-01T00:00:00") {
if t.Valid && (t.Val.IsZero() || t.Val.Format(time.RFC3339) == "0001-01-01T00:00:00Z") {
t.Valid = false
}
return nil
}
func (t SqlTimeStamp) Value() (driver.Value, error) {
if !t.Valid || t.Val.IsZero() || t.Val.Before(time.Date(0002, 1, 1, 0, 0, 0, 0, time.UTC)) {
if !t.Valid || t.Val.IsZero() || t.Val.Before(time.Date(0o002, 1, 1, 0, 0, 0, 0, time.UTC)) {
return nil, nil
}
return t.Val.Format("2006-01-02T15:04:05"), nil
return t.Val.Format(time.RFC3339), nil
}
func (t SqlTimeStamp) MarshalYAML() (any, error) {
if !t.Valid || t.Val.IsZero() || t.Val.Before(time.Date(0002, 1, 1, 0, 0, 0, 0, time.UTC)) {
if !t.Valid || t.Val.IsZero() || t.Val.Before(time.Date(0o002, 1, 1, 0, 0, 0, 0, time.UTC)) {
return nil, nil
}
return t.Val.Format("2006-01-02T15:04:05"), nil
return t.Val.Format(time.RFC3339), nil
}
func (t *SqlTimeStamp) UnmarshalYAML(value *yaml.Node) error {
if err := t.SqlNull.UnmarshalYAML(value); err != nil {
return err
}
if t.Valid && (t.Val.IsZero() || t.Val.Format("2006-01-02T15:04:05") == "0001-01-01T00:00:00") {
if t.Valid && (t.Val.IsZero() || t.Val.Format(time.RFC3339) == "0001-01-01T00:00:00Z") {
t.Valid = false
}
return nil
}
func (t SqlTimeStamp) MarshalXML(e *xml.Encoder, start xml.StartElement) error {
if !t.Valid || t.Val.IsZero() || t.Val.Before(time.Date(0002, 1, 1, 0, 0, 0, 0, time.UTC)) {
if !t.Valid || t.Val.IsZero() || t.Val.Before(time.Date(0o002, 1, 1, 0, 0, 0, 0, time.UTC)) {
return e.EncodeElement("", start)
}
return e.EncodeElement(t.Val.Format("2006-01-02T15:04:05"), start)
return e.EncodeElement(t.Val.Format(time.RFC3339), start)
}
func (t *SqlTimeStamp) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error {
@@ -510,7 +511,7 @@ func (t *SqlTimeStamp) UnmarshalXML(d *xml.Decoder, start xml.StartElement) erro
return err
}
t.Val = tm
t.Valid = !tm.IsZero() && tm.Format("2006-01-02T15:04:05") != "0001-01-01T00:00:00"
t.Valid = !tm.IsZero() && tm.Format(time.RFC3339) != "0001-01-01T00:00:00Z"
return nil
}
+1 -2
View File
@@ -178,7 +178,7 @@ func TestSqlTimeStamp_JSON(t *testing.T) {
if err != nil {
t.Fatalf("Marshal failed: %v", err)
}
expected := `"2024-01-15T10:30:45"`
expected := `"2024-01-15T10:30:45Z"`
if string(data) != expected {
t.Errorf("expected %s, got %s", expected, string(data))
}
@@ -955,4 +955,3 @@ func TestSqlByteArray_Base64_RoundTrip(t *testing.T) {
t.Errorf("Round-trip failed: expected %v, got %v", original, b3.Val)
}
}
+1 -1
View File
@@ -195,7 +195,7 @@ func TestSqlTimeStamp_YAML(t *testing.T) {
if err != nil {
t.Fatalf("Marshal failed: %v", err)
}
if string(data) != "2024-06-15T09:30:00\n" {
if string(data) != "\"2024-06-15T09:30:00Z\"\n" {
t.Errorf("unexpected YAML: %q", string(data))
}
var ts2 SqlTimeStamp
+4
View File
@@ -207,6 +207,10 @@ func (se *SchemaEditor) showDomainEditor(index int, domain *models.Domain) {
se.showDomainList()
})
form.AddButton("Tables", func() {
se.showDomainTables(index)
})
form.AddButton("Delete", func() {
se.showDeleteDomainConfirm(index)
})
+12
View File
@@ -39,6 +39,18 @@ func (se *SchemaEditor) createMainMenu() tview.Primitive {
AddItem("Manage Domains", "View, create, edit, and delete domains", 'd', func() {
se.showDomainList()
}).
AddItem("Manage Indexes", "View, create, edit, and delete table indexes", 'x', func() {
se.showObjectList(se.indexKind())
}).
AddItem("Manage Views", "View, create, edit, and delete views", 'v', func() {
se.showObjectList(se.viewKind())
}).
AddItem("Manage Sequences", "View, create, edit, and delete sequences", 'u', func() {
se.showObjectList(se.sequenceKind())
}).
AddItem("Manage Scripts", "View, create, edit, and delete SQL scripts", 'c', func() {
se.showObjectList(se.scriptKind())
}).
AddItem("Import & Merge", "Import and merge schema from another database", 'i', func() {
se.showImportScreen()
}).
+263
View File
@@ -0,0 +1,263 @@
package ui
import (
"errors"
"fmt"
"strings"
"git.warky.dev/wdevs/relspecgo/pkg/models"
)
// Data operations for indexes, views, sequences, scripts and domain/table assignment.
func (se *SchemaEditor) schemaAt(schemaIndex int) (*models.Schema, error) {
if schemaIndex < 0 || schemaIndex >= len(se.db.Schemas) {
return nil, errors.New("schema not found")
}
return se.db.Schemas[schemaIndex], nil
}
func (se *SchemaEditor) tableAt(schemaIndex, tableIndex int) (*models.Schema, *models.Table, error) {
schema, err := se.schemaAt(schemaIndex)
if err != nil {
return nil, nil, err
}
if tableIndex < 0 || tableIndex >= len(schema.Tables) {
return nil, nil, errors.New("table not found")
}
return schema, schema.Tables[tableIndex], nil
}
// splitList splits a comma separated list, trimming blanks and dropping empty entries.
func splitList(s string) []string {
parts := make([]string, 0)
for _, p := range strings.Split(s, ",") {
if p = strings.TrimSpace(p); p != "" {
parts = append(parts, p)
}
}
return parts
}
// SaveIndex adds an index to a table. When oldName is non-empty the index of that
// name is replaced (and renamed if needed).
func (se *SchemaEditor) SaveIndex(schemaIndex, tableIndex int, oldName string, idx *models.Index) error {
schema, table, err := se.tableAt(schemaIndex, tableIndex)
if err != nil {
return err
}
idx.Name = strings.TrimSpace(idx.Name)
if idx.Name == "" {
return errors.New("index name is required")
}
if len(idx.Columns) == 0 {
return errors.New("index needs at least one column")
}
for _, c := range idx.Columns {
if _, ok := table.Columns[c]; !ok {
return fmt.Errorf("column %q not found in table %s", c, table.Name)
}
}
if _, exists := table.Indexes[idx.Name]; exists && idx.Name != oldName {
return fmt.Errorf("index %q already exists", idx.Name)
}
if table.Indexes == nil {
table.Indexes = make(map[string]*models.Index)
}
if oldName != "" {
delete(table.Indexes, oldName)
}
idx.Table = table.Name
idx.Schema = schema.Name
table.Indexes[idx.Name] = idx
table.UpdateDate()
se.db.UpdateDate()
return nil
}
// DeleteIndex removes an index from a table.
func (se *SchemaEditor) DeleteIndex(schemaIndex, tableIndex int, name string) bool {
_, table, err := se.tableAt(schemaIndex, tableIndex)
if err != nil {
return false
}
if _, ok := table.Indexes[name]; !ok {
return false
}
delete(table.Indexes, name)
table.UpdateDate()
se.db.UpdateDate()
return true
}
// SaveView adds a view to a schema, or replaces the one at position at (use -1 to add).
func (se *SchemaEditor) SaveView(schemaIndex, at int, v *models.View) error {
schema, err := se.schemaAt(schemaIndex)
if err != nil {
return err
}
v.Name = strings.TrimSpace(v.Name)
if v.Name == "" {
return errors.New("view name is required")
}
if strings.TrimSpace(v.Definition) == "" {
return errors.New("view definition is required")
}
for i, o := range schema.Views {
if i != at && o.Name == v.Name {
return fmt.Errorf("view %q already exists", v.Name)
}
}
v.Schema = schema.Name
if at >= 0 && at < len(schema.Views) {
schema.Views[at] = v
} else {
schema.Views = append(schema.Views, v)
}
schema.UpdateDate()
se.db.UpdateDate()
return nil
}
// DeleteView removes the view at position at.
func (se *SchemaEditor) DeleteView(schemaIndex, at int) bool {
schema, err := se.schemaAt(schemaIndex)
if err != nil || at < 0 || at >= len(schema.Views) {
return false
}
schema.Views = append(schema.Views[:at], schema.Views[at+1:]...)
schema.UpdateDate()
se.db.UpdateDate()
return true
}
// SaveSequence adds a sequence to a schema, or replaces the one at position at (use -1 to add).
func (se *SchemaEditor) SaveSequence(schemaIndex, at int, s *models.Sequence) error {
schema, err := se.schemaAt(schemaIndex)
if err != nil {
return err
}
s.Name = strings.TrimSpace(s.Name)
if s.Name == "" {
return errors.New("sequence name is required")
}
if s.IncrementBy == 0 {
return errors.New("increment must not be zero")
}
for i, o := range schema.Sequences {
if i != at && o.Name == s.Name {
return fmt.Errorf("sequence %q already exists", s.Name)
}
}
s.Schema = schema.Name
if at >= 0 && at < len(schema.Sequences) {
schema.Sequences[at] = s
} else {
schema.Sequences = append(schema.Sequences, s)
}
schema.UpdateDate()
se.db.UpdateDate()
return nil
}
// DeleteSequence removes the sequence at position at.
func (se *SchemaEditor) DeleteSequence(schemaIndex, at int) bool {
schema, err := se.schemaAt(schemaIndex)
if err != nil || at < 0 || at >= len(schema.Sequences) {
return false
}
schema.Sequences = append(schema.Sequences[:at], schema.Sequences[at+1:]...)
schema.UpdateDate()
se.db.UpdateDate()
return true
}
// SaveScript adds a script to a schema, or replaces the one at position at (use -1 to add).
func (se *SchemaEditor) SaveScript(schemaIndex, at int, s *models.Script) error {
schema, err := se.schemaAt(schemaIndex)
if err != nil {
return err
}
s.Name = strings.TrimSpace(s.Name)
if s.Name == "" {
return errors.New("script name is required")
}
if strings.TrimSpace(s.SQL) == "" {
return errors.New("script SQL is required")
}
for i, o := range schema.Scripts {
if i != at && o.Name == s.Name {
return fmt.Errorf("script %q already exists", s.Name)
}
}
s.Schema = schema.Name
if at >= 0 && at < len(schema.Scripts) {
schema.Scripts[at] = s
} else {
schema.Scripts = append(schema.Scripts, s)
}
schema.UpdateDate()
se.db.UpdateDate()
return nil
}
// DeleteScript removes the script at position at.
func (se *SchemaEditor) DeleteScript(schemaIndex, at int) bool {
schema, err := se.schemaAt(schemaIndex)
if err != nil || at < 0 || at >= len(schema.Scripts) {
return false
}
schema.Scripts = append(schema.Scripts[:at], schema.Scripts[at+1:]...)
schema.UpdateDate()
se.db.UpdateDate()
return true
}
// AssignTableToDomain adds a reference to schemaName.tableName to the domain at domainIndex.
func (se *SchemaEditor) AssignTableToDomain(domainIndex int, schemaName, tableName string) error {
if domainIndex < 0 || domainIndex >= len(se.db.Domains) {
return errors.New("domain not found")
}
domain := se.db.Domains[domainIndex]
var table *models.Table
for _, s := range se.db.Schemas {
if s.Name != schemaName {
continue
}
for _, t := range s.Tables {
if t.Name == tableName {
table = t
}
}
}
if table == nil {
return fmt.Errorf("table %s.%s not found", schemaName, tableName)
}
for _, dt := range domain.Tables {
if dt.SchemaName == schemaName && dt.TableName == tableName {
return fmt.Errorf("table %s.%s is already in domain %s", schemaName, tableName, domain.Name)
}
}
dt := models.InitDomainTable(tableName, schemaName)
dt.RefTable = table
dt.Sequence = uint(len(domain.Tables))
domain.Tables = append(domain.Tables, dt)
se.db.UpdateDate()
return nil
}
// UnassignTableFromDomain removes the reference to schemaName.tableName from the domain.
func (se *SchemaEditor) UnassignTableFromDomain(domainIndex int, schemaName, tableName string) bool {
if domainIndex < 0 || domainIndex >= len(se.db.Domains) {
return false
}
domain := se.db.Domains[domainIndex]
for i, dt := range domain.Tables {
if dt.SchemaName == schemaName && dt.TableName == tableName {
domain.Tables = append(domain.Tables[:i], domain.Tables[i+1:]...)
se.db.UpdateDate()
return true
}
}
return false
}
+136
View File
@@ -0,0 +1,136 @@
package ui
import (
"testing"
"git.warky.dev/wdevs/relspecgo/pkg/models"
)
func newTestEditor() *SchemaEditor {
db := models.InitDatabase("test")
schema := models.InitSchema("public")
table := models.InitTable("users", "public")
table.Columns["id"] = models.InitColumn("id", "users", "public")
table.Columns["email"] = models.InitColumn("email", "users", "public")
schema.Tables = append(schema.Tables, table)
db.Schemas = append(db.Schemas, schema)
return &SchemaEditor{db: db}
}
func TestSaveIndex(t *testing.T) {
se := newTestEditor()
table := se.db.Schemas[0].Tables[0]
tests := []struct {
name string
old string
idx *models.Index
wantErr bool
}{
{"valid", "", &models.Index{Name: "idx_email", Columns: []string{"email"}, Unique: true}, false},
{"duplicate", "", &models.Index{Name: "idx_email", Columns: []string{"email"}}, true},
{"missing name", "", &models.Index{Columns: []string{"email"}}, true},
{"no columns", "", &models.Index{Name: "idx_none"}, true},
{"unknown column", "", &models.Index{Name: "idx_bad", Columns: []string{"nope"}}, true},
{"rename", "idx_email", &models.Index{Name: "idx_email2", Columns: []string{"email", "id"}}, false},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if err := se.SaveIndex(0, 0, tt.old, tt.idx); (err != nil) != tt.wantErr {
t.Fatalf("err = %v, wantErr %v", err, tt.wantErr)
}
})
}
if _, ok := table.Indexes["idx_email"]; ok {
t.Error("renamed index should be gone under old name")
}
if idx := table.Indexes["idx_email2"]; idx == nil || idx.Table != "users" || idx.Schema != "public" {
t.Errorf("unexpected renamed index: %+v", idx)
}
if !se.DeleteIndex(0, 0, "idx_email2") || se.DeleteIndex(0, 0, "idx_email2") {
t.Error("delete should succeed once")
}
}
func TestSaveViewSequenceScript(t *testing.T) {
se := newTestEditor()
schema := se.db.Schemas[0]
if err := se.SaveView(0, -1, &models.View{Name: "v", Definition: "select 1"}); err != nil {
t.Fatal(err)
}
if err := se.SaveView(0, -1, &models.View{Name: "v", Definition: "select 2"}); err == nil {
t.Error("duplicate view accepted")
}
if err := se.SaveView(0, 0, &models.View{Name: "v", Definition: "select 3"}); err != nil {
t.Errorf("editing in place should not conflict: %v", err)
}
if err := se.SaveView(0, -1, &models.View{Name: "w"}); err == nil {
t.Error("view without definition accepted")
}
if len(schema.Views) != 1 || schema.Views[0].Definition != "select 3" || schema.Views[0].Schema != "public" {
t.Errorf("unexpected views: %+v", schema.Views)
}
if !se.DeleteView(0, 0) || se.DeleteView(0, 0) {
t.Error("view delete mismatch")
}
if err := se.SaveSequence(0, -1, &models.Sequence{Name: "s", IncrementBy: 1, StartValue: 1}); err != nil {
t.Fatal(err)
}
if err := se.SaveSequence(0, -1, &models.Sequence{Name: "z"}); err == nil {
t.Error("zero increment accepted")
}
if !se.DeleteSequence(0, 0) || len(schema.Sequences) != 0 {
t.Error("sequence delete failed")
}
if err := se.SaveScript(0, -1, &models.Script{Name: "init", SQL: "select 1"}); err != nil {
t.Fatal(err)
}
if err := se.SaveScript(0, -1, &models.Script{Name: "empty"}); err == nil {
t.Error("script without SQL accepted")
}
if err := se.SaveScript(5, -1, &models.Script{Name: "x", SQL: "y"}); err == nil {
t.Error("bad schema index accepted")
}
if !se.DeleteScript(0, 0) || len(schema.Scripts) != 0 {
t.Error("script delete failed")
}
}
func TestDomainTableAssignment(t *testing.T) {
se := newTestEditor()
se.createDomainNoUI("core")
if err := se.AssignTableToDomain(0, "public", "users"); err != nil {
t.Fatal(err)
}
if err := se.AssignTableToDomain(0, "public", "users"); err == nil {
t.Error("duplicate assignment accepted")
}
if err := se.AssignTableToDomain(0, "public", "missing"); err == nil {
t.Error("unknown table accepted")
}
if err := se.AssignTableToDomain(3, "public", "users"); err == nil {
t.Error("bad domain index accepted")
}
dt := se.db.Domains[0].Tables[0]
if dt.RefTable != se.db.Schemas[0].Tables[0] {
t.Error("RefTable not linked")
}
if !se.UnassignTableFromDomain(0, "public", "users") || se.UnassignTableFromDomain(0, "public", "users") {
t.Error("unassign mismatch")
}
}
func (se *SchemaEditor) createDomainNoUI(name string) {
se.db.Domains = append(se.db.Domains, models.InitDomain(name))
}
func TestSplitList(t *testing.T) {
got := splitList(" a, b,, c ,")
if len(got) != 3 || got[0] != "a" || got[2] != "c" {
t.Errorf("got %v", got)
}
}

Some files were not shown because too many files have changed in this diff Show More