Compare commits

..
Author SHA1 Message Date
SG Command 778379538b fix: make MySQL writer execute generated DDL 2026-10-03 12:42:07 +02:00
SG Command 7a9219b6e3 feat: add MySQL reader and writer 2026-10-03 12:36:39 +02:00
warkanum b38f53c603 docs(tests): add test coverage plans 2026-10-03 10:01:48 +02:00
warkanum ccba53c494 test: add podman/docker dbtest tool for postgres, mssql and mysql 2026-10-03 10:01:48 +02:00
sgcommand 53327b9a5a feat(ui): TUI indexes, views, sequences, scripts, domain/table assignment (#40) (#43)
Co-authored-by: SG Command <sgcommand@warky.dev>
2026-10-03 07:27:25 +00:00
sgcommand 734b14d48d docs: add usage examples for each format combination (#42)
Co-authored-by: SG Command <sgcommand@warky.dev>
2026-10-03 07:27:03 +00:00
sgcommand 938f0ed51f feat(cli): add --dry-run to convert, merge and split (#41)
Co-authored-by: SG Command <sgcommand@warky.dev>
2026-10-03 07:26:54 +00:00
warkanum 6e2e7eb19e feat(release): add rerelease target to move latest tag 2026-10-02 23:41:42 +02:00
warkanum 82e86cdf39 fix(release): run lint and format check before version release
Release / test (push) Successful in 2m7s
Release / release (push) Successful in 1m49s
Release / pkg-aur (push) Successful in 50s
Release / pkg-deb (push) Successful in 1m30s
Release / pkg-rpm (push) Successful in 1m48s
2026-10-02 23:38:28 +02:00
warkanum b7ae00ab52 fix(test): simplify parameter list in identity column builder 2026-10-02 23:35:56 +02:00
warkanum 8f8664357e chore(release): update package version to 1.0.85
Release / test (push) Failing after 2m1s
Release / release (push) Skipped
Release / pkg-aur (push) Skipped
Release / pkg-deb (push) Skipped
Release / pkg-rpm (push) Skipped
2026-10-02 23:32:07 +02:00
warkanum 7bd61a21b5 fix: resolve gofumpt and staticcheck lint errors 2026-10-02 23:31:41 +02:00
warkanum 6f6b9834ca docs(CONTRIBUTING): update setup instructions and add make targets 2026-10-02 23:29:25 +02:00
warkanum 9cc10715e3 feat: generated and identity columns for mssql, sqlite, drizzle, typeorm, bun and gorm readers
- mssql: write computed columns (AS (expr) PERSISTED) and identity; read computed definition and identity
- sqlite: write GENERATED ALWAYS AS (expr) STORED; read generated columns via table_xinfo and parse the expression
- drizzle: generatedAlwaysAs / generated(Always|ByDefault)AsIdentity, read and write
- typeorm: asExpression/generatedType and identity decorators, read and write; decorator scan is now quote-aware
- bun, gorm readers: read generated and identity markers
- readmes updated
2026-10-02 23:04:04 +02:00
warkanum bbab5ce936 feat: mark generated and identity columns across writers and DBML
- bun: scanonly+generated for GENERATED columns; scanonly+identity for ALWAYS identity (PK gets identity only)
- gorm: <-:false+generated / identity, same primary key rule
- dbml: write GENERATED/IDENTITY column notes and read them back into the model
- pgsql: emit generation/identity clauses in migration create-table and add-column, no DEFAULT
- readmes updated
2026-10-02 22:58:00 +02:00
warkanum 275424c605 feat(pgsql): diff against the live database by default for direct output
Direct pgsql output (job output.conn_env, merge --output-conn) now reads the live
schema and executes only the differences. full_ddl: true (job option) or --full-ddl
(merge) restores the full idempotent DDL; file output is unchanged.

- compare PKs by columns, skip constraint-backed indexes, normalize index method,
  FK actions, serial/numeric types, default literals/casts and truncated names
- diff table and column comments instead of re-emitting them
- remove leftover ZZDUMP debug code
2026-10-02 22:52:59 +02:00
warkanum b6c0cd3d1b fix(pgsql-reader): pair composite foreign key columns by position
key_column_usage and constraint_column_usage were joined on constraint name only,
so an N-column foreign key returned N*N column pairs. Read from pg_constraint with
unnest(conkey, confkey) instead.
2026-10-02 22:52:53 +02:00
warkanum d7d1d99ebc fix(pgsql): tie PK sequence to nextval default, setval past data, keep serial defaults
- derive the primary key sequence from the column's nextval() default instead of an unused identity_<table>_<pk> sequence
- setval after table creation (full and diff paths), forward-only, MAX+1 with is_called=false
- do not DROP DEFAULT on serial/bigserial columns without a model default
2026-10-02 22:37:37 +02:00
warkanum 43849324ce chore(release): update package version to 1.0.84
Release / test (push) Successful in 1m25s
Release / release (push) Successful in 2m7s
Release / pkg-aur (push) Successful in 55s
Release / pkg-deb (push) Successful in 1m42s
Release / pkg-rpm (push) Successful in 1m49s
2026-09-24 15:41:46 +02:00
warkanum e1df3b7cee fix(writers): drop version stamps from generated output
Generated files carried the RelSpec version and build date, so every
rebuild produced a diff even when the schema was unchanged.

- bun, gorm, drizzle: remove the GeneratedBy template field and line
- pgsql, mssql, sqlite: emit a constant "-- Generated by RelSpec"
- bun: replace the version header test with a version-free assertion

pkg/buildinfo is unchanged and still backs the CLI version.
2026-09-24 15:40:37 +02:00
warkanum f7b5d5f054 chore(release): update package version to 1.0.83
Release / pkg-deb (push) Successful in 1m47s
Release / test (push) Successful in 59s
Release / release (push) Successful in 2m0s
Release / pkg-aur (push) Successful in 54s
Release / pkg-rpm (push) Successful in 1m56s
2026-09-23 18:52:48 +02:00
warkanum 99fc4b0944 Merge pull request 'Fix/dbml commented refs duplicate indexes' (#31) from fix/dbml-commented-refs-duplicate-indexes into master
Reviewed-on: #31
2026-09-23 16:48:54 +00:00
warkanum 278d488363 chore(lint): use strings.EqualFold in jobs output format check 2026-09-23 18:48:07 +02:00
warkanum 2f69205aa0 fix(dbml): resolve commented cross-file // Ref: lines
Commented refs are collected per file and resolved against the combined
model after all inputs are loaded (directory, --from-list, merge, jobs).
Matched refs become FKs and relationships; duplicates of existing FKs are
skipped; missing targets are skipped with a warning; column type
mismatches warn.

Also keep reused index names within a DBML table instead of overwriting,
give a second FK to the same table a distinct relationship name, and make
the pgsql writer match relationships to FKs by name first.
2026-09-23 18:48:07 +02:00
warkanum b91985c493 feat(inspector): fail on duplicate index names per schema
PostgreSQL index names share the schema-wide relation namespace, so a
reused name makes CREATE INDEX IF NOT EXISTS silently skip the duplicate.
Add duplicate_index_name rule (enforce by default) covering indexes and
PK/unique constraint backing indexes.
2026-09-23 18:48:07 +02:00
warkanum 80a3453233 Merge pull request 'fix(jobs): preserve writer and template output options' (#30) from issue-29-job-manifest-options into master
Reviewed-on: #30
Reviewed-by: Warky <2+warkanum@noreply@warky.dev>
2026-09-21 04:00:50 +00:00
SG Command ade71b75fa fix(jobs): preserve writer and template output options 2026-09-20 23:16:41 +02:00
warkanum 799b7feb53 ci(rpm): update RPM builder to avoid transient mirrors 2026-09-20 21:41:52 +02:00
warkanum eb40201b59 chore(release): update package version to 1.0.82
Release / release (push) Successful in 1m50s
Release / pkg-aur (push) Successful in 1m0s
Release / pkg-deb (push) Successful in 1m17s
Release / test (push) Successful in 53s
Release / pkg-rpm (push) Failing after 16s
2026-09-20 20:37:41 +02:00
warkanum 1461f8f69f chore(release): update version to 1.0.81 and metadata 2026-09-20 20:37:09 +02:00
warkanum ea70e19a46 feat(jobs): expand environment variables in paths 2026-09-20 20:08:20 +02:00
warkanum 52f4642d97 feat(models): add support for generated and identity columns
Release / test (push) Successful in 3m13s
Release / release (push) Successful in 1m52s
Release / pkg-aur (push) Successful in 1m39s
Release / pkg-deb (push) Successful in 2m15s
Release / pkg-rpm (push) Failing after 46s
* extend Column struct with Generated, GenerationExpression, Identity, and IdentityGeneration fields
* update queryColumns to handle new column properties
* modify migration writer to avoid altering generated and identity columns
* add tests to ensure correct handling of generated and identity columns
2026-09-20 18:33:12 +02:00
warkanum a84592374c Merge pull request 'Feat generated headers column order' (#28) from feat-generated-headers-column-order into master
Release / pkg-rpm (push) Successful in 3m58s
Release / release (push) Successful in 1m54s
Release / test (push) Successful in 5m49s
Release / pkg-aur (push) Successful in 7m13s
Release / pkg-deb (push) Successful in 5m8s
Reviewed-on: #28
2026-09-10 20:22:50 +00:00
warkanum ca226e83df feat(release): update release process to include linting and testing 2026-09-10 22:21:40 +02:00
warkanum 19a2cc1fe3 feat(writers): stamp RelSpec version in generated headers; preserve DBML column order
- Add pkg/buildinfo with Version/BuildDate (set via ldflags, VCS fallback);
  cmd/relspec now sources version info from it
- Emit "RelSpec <version> (built: <date>)" in generated file headers for
  bun, gorm, drizzle, pgsql (incl. migration), mssql, sqlite writers
- pgsql writer: getSortedColumns now sorts by Sequence then Name so the
  streaming WriteSchema/WriteDatabase path preserves source column order
- dbml reader: mergeTable re-bases merged-in column Sequence values past the
  existing max, fixing colliding sequences (and alphabetical fallback) when a
  table is split across multiple DBML files
- Tests for bun/gorm header + column order, and dbml multi-file merge ordering
2026-09-10 22:08:49 +02:00
warkanum 58e46e5b59 feat(dbml): support case-insensitive table note parsing
Release / test (push) Successful in 4m15s
Release / release (push) Successful in 5m29s
Release / pkg-rpm (push) Successful in 2m10s
Release / pkg-deb (push) Successful in 2m27s
Release / pkg-aur (push) Successful in 5m52s
2026-09-09 22:31:00 +02:00
warkanum 161ac317f0 feat(dbml): support multiline table notes in DBML
Release / test (push) Successful in 1m45s
Release / release (push) Successful in 12m24s
Release / pkg-rpm (push) Successful in 2m3s
Release / pkg-deb (push) Successful in 2m17s
Release / pkg-aur (push) Successful in 2m47s
* Add parsing for triple-quoted table notes in DBML
* Update writer to format multiline notes correctly
* Enhance tests for multiline table notes handling
2026-09-09 21:38:11 +02:00
warkanum ee5c009234 feat(main): add silent mode to suppress output messages
* implement --silent flag to control output verbosity
* update error handling to restore stderr after silent mode
* enhance progress reporting in PostgreSQL reader
2026-09-09 21:30:15 +02:00
warkanum 9e9a17d578 Merge pull request 'feat(dbml): @postgres/@sqlite dialect directives (#19)' (#27) from issue-19-dbml-directives into master
Release / test (push) Successful in 58s
Release / release (push) Successful in 5m13s
Release / pkg-aur (push) Successful in 1m0s
Release / pkg-rpm (push) Successful in 2m55s
Release / pkg-deb (push) Successful in 3m21s
Reviewed-on: #27
2026-09-08 14:19:02 +00:00
warkanum 7b628b888c Merge pull request 'feat(job): complete deferred job-file features (#20)' (#26) from issue-20-complete-job-files into master
Reviewed-on: #26
2026-09-08 14:18:57 +00:00
Hein 84a6b31873 feat(job): complete deferred job-file features
Implements the remaining items from issue #20:

- version is now forward-permissive: any value >= 1 is accepted; a
  newer-than-known version loads best-effort (unknown fields ignored,
  warning printed) instead of hard-failing on "must be 1"
- from_job input reference: `inputs: [{ from_job: <job> }]` resolves to
  that job's single-file output + format and implies a dependency edge;
  combined depends_on + from_job graph gets topological ordering and
  cycle detection
- logfile size-rotation, on by default (5MB, keep 3), overridable per
  job (log_max_size / log_keep) or file-wide via a top-level defaults block
- new commands: split (schema/table subsetting via select:), inspect
  (rule validation -> markdown/json report, fails job on enforced-rule
  errors), diff (compare exactly two schemas, never fails), scripts-exec
  (run SQL script dirs against a live PostgreSQL database)
- atomic single-file output/report writes (temp file + rename)
- symlink-escape hardening in SafeJoin via EvalSymlinks preflight

Updates docs/JOB_FILES.md and examples/jobs/relspec.yml accordingly.
2026-09-08 14:32:07 +02:00
180 changed files with 20168 additions and 497 deletions
+12 -3
View File
@@ -63,6 +63,8 @@ jobs:
- name: Build release binaries
run: |
VERSION="${{ github.event.inputs.tag || github.ref_name }}"
BUILD_DATE="$(date -u +"%Y-%m-%d %H:%M:%S UTC")"
LDFLAGS="-X 'git.warky.dev/wdevs/relspecgo/pkg/buildinfo.Version=${VERSION}' -X 'git.warky.dev/wdevs/relspecgo/pkg/buildinfo.BuildDate=${BUILD_DATE}'"
for target in "linux/amd64" "linux/arm64" "darwin/amd64" "darwin/arm64" "windows/amd64"; do
GOOS="${target%/*}"
GOARCH="${target#*/}"
@@ -71,7 +73,7 @@ jobs:
NAME="relspec-${GOOS}-${GOARCH}${EXT}"
GOOS="$GOOS" GOARCH="$GOARCH" go build \
-trimpath \
-ldflags "-X git.warky.dev/wdevs/relspecgo/cmd/relspec.version=${VERSION}" \
-ldflags "$LDFLAGS" \
-o "$NAME" ./cmd/relspec
echo "Built $NAME"
done
@@ -234,11 +236,12 @@ jobs:
run: |
VERSION="${{ github.event.inputs.tag || github.ref_name }}"
PKGVER="${VERSION#v}"
BUILD_DATE="$(date -u +"%Y-%m-%d %H:%M:%S UTC")"
for GOARCH in amd64 arm64; do
GOOS=linux GOARCH=$GOARCH go build \
-trimpath \
-ldflags "-X git.warky.dev/wdevs/relspecgo/cmd/relspec.version=${PKGVER}" \
-ldflags "-X 'git.warky.dev/wdevs/relspecgo/pkg/buildinfo.Version=v${PKGVER}' -X 'git.warky.dev/wdevs/relspecgo/pkg/buildinfo.BuildDate=${BUILD_DATE}'" \
-o relspec ./cmd/relspec
PKGDIR="relspec_${PKGVER}_${GOARCH}"
@@ -309,7 +312,13 @@ jobs:
rockylinux:9 \
bash -lc "
set -euo pipefail
dnf install -y rpm-build git &&
# Avoid transient/out-of-sync mirrors while bootstrapping the RPM builder.
sed -i \
-e 's|^mirrorlist=|#mirrorlist=|' \
-e 's|^#baseurl=http://dl.rockylinux.org|baseurl=https://dl.rockylinux.org|' \
/etc/yum.repos.d/rocky*.repo
dnf clean all
dnf -y --setopt=timeout=60 --setopt=retries=5 install rpm-build git
curl -fsSL https://go.dev/dl/go\${GO_VER}.linux-amd64.tar.gz | tar -C /usr/local -xz &&
export PATH=\$PATH:/usr/local/go/bin &&
mkdir -p ~/rpmbuild/{BUILD,BUILDROOT,RPMS,SOURCES,SPECS,SRPMS} &&
+59 -113
View File
@@ -1,145 +1,91 @@
# Contributing to RelSpec
Thank you for your interest in contributing to RelSpec.
## Setup
## Development Setup
- Go 1.25+ (see `go.mod`), Git
- Optional: golangci-lint, Docker/Podman (PostgreSQL integration tests)
### Prerequisites
- Go 1.21 or higher
- Git
- (Optional) golangci-lint for linting
- (Optional) Docker for database testing
### Getting Started
1. Clone the repository:
```bash
git clone https://github.com/wdevs/relspecgo.git
git clone git@git.warky.dev:wdevs/relspecgo.git
cd relspecgo
make deps
make build # outputs build/relspec
```
2. Install dependencies:
```bash
go mod download
```
## Make Targets
3. Run tests:
```bash
go test ./...
```
| Target | Purpose |
|-------------------------|------------------------------------------------------|
| `make test` | Unit tests (race detection, coverage) |
| `make test-integration` | Integration tests (needs `RELSPEC_TEST_PG_CONN`) |
| `make docker-test` | PostgreSQL integration tests via Docker/Podman |
| `make lint` | golangci-lint |
| `make fmt` / `fmt-check`| gofumpt + goimports |
| `make check` | vet, fmt-check, staticcheck, govulncheck |
| `make coverage` | Coverage report |
4. Build the project:
```bash
go build -o relspec ./cmd/relspec
```
Single test: `go test -run TestName ./pkg/readers/dbml`
## Project Structure
## Layout
```
relspecgo/
├── cmd/ # CLI application entry point
├── pkg/
│ ├── readers/ # Input format readers (XML, JSON, DCTX, DB, GORM, Bun)
│ ├── writers/ # Output format writers (GORM, Bun, JSON, YAML)
│ ├── models/ # Internal data models for relations
│ └── transform/ # Transformation and validation logic
├── examples/ # Usage examples and sample files
├── tests/ # Integration tests
└── .claude/ # Claude Code configuration and commands
cmd/relspec/ CLI commands (convert, diff, merge, split, edit, inspect, job, ...)
pkg/models/ Core model: Database > Schema > Table > Column/Constraint/Index/Relationship
pkg/readers/<fmt> One reader per format
pkg/writers/<fmt> One writer per format
pkg/diff, merge, inspector, jobs, transform, ui, pgsql, sqltypes ...
examples/ Sample files
tests/ Integration tests and assets
docs/ Feature docs
```
## Adding New Readers
## Adding a Reader
To add a new input format reader:
1. Create `pkg/readers/<format>/reader.go` with `NewReader(options *readers.ReaderOptions)`.
2. Implement `readers.Reader`: `ReadDatabase`, `ReadSchema`, `ReadTable`.
3. Add `reader_test.go` in the same package.
4. Register the format in the CLI switches (`cmd/relspec/convert.go`, `diff.go`, `edit.go`, ...).
5. Add a `README.md` in the reader directory.
1. Create a new file in `pkg/readers/` (e.g., `myformat_reader.go`)
2. Implement the `Reader` interface:
```go
type Reader interface {
Read(source string) (*models.Schema, error)
}
```
3. Add tests in `pkg/readers/myformat_reader_test.go`
4. Register the reader in the CLI
## Adding a Writer
## Adding New Writers
1. Create `pkg/writers/<format>/writer.go` with `NewWriter(options *writers.WriterOptions)`.
2. Implement `writers.Writer`: `WriteDatabase`, `WriteSchema`, `WriteTable`.
3. Add `writer_test.go` in the same package.
4. Register the format in the CLI switches.
5. Add a `README.md` in the writer directory.
To add a new output format writer:
## Code Rules
1. Create a new file in `pkg/writers/` (e.g., `myformat_writer.go`)
2. Implement the `Writer` interface:
```go
type Writer interface {
Write(schema *models.Schema, destination string) error
}
```
3. Add tests in `pkg/writers/myformat_writer_test.go`
4. Register the writer in the CLI
## Code Style
- Follow standard Go conventions
- Use `gofmt` for formatting
- Run `go vet` to check for issues
- Use meaningful variable and function names
- Add comments for exported functions and types
- Format with gofumpt/goimports (`make fmt`); `make check` must pass.
- Iterate `Table.Columns`, `Constraints`, `Indexes`, `Relationships` in sorted order (maps are unordered; output must be deterministic).
- Every writer stamps `buildinfo.GeneratedComment()` in its file header.
- Comment exported functions and types.
## Testing
- Write unit tests for all new functionality
- Aim for >80% code coverage
- Use table-driven tests where appropriate
- Include both positive and negative test cases
- Tests live in the same package as the code.
- Table-driven tests; cover positive and negative cases.
- Reuse existing test data in `tests/` and `examples/` before adding new data.
- Tests run with `-race`.
### Running Tests
## Commits
```bash
# All tests
go test ./...
- Format: `type(scope): description`
- Types: `feat`, `fix`, `docs`, `test`, `refactor`, `chore`, `ci`
- Keep commits focused. Reference issues where applicable.
# With coverage
go test -cover ./...
## Pull Requests
# Verbose output
go test -v ./...
1. Branch from `master`.
2. Add tests; `make test` and `make check` pass.
3. Update docs/README if behaviour changes.
4. Open a PR with a clear description.
# Specific package
go test ./pkg/readers/...
```
## Security
## Committing Changes
- Write clear, descriptive commit messages
- Follow conventional commits format: `type(scope): description`
- Types: feat, fix, docs, test, refactor, chore
- Example: `feat(readers): add PostgreSQL support`
- Keep commits focused and atomic
- Reference issues in commit messages when applicable
## Pull Request Process
1. Create a feature branch from `master`
2. Make your changes
3. Add tests for new functionality
4. Ensure all tests pass
5. Update documentation if needed
6. Submit a pull request with a clear description
## Claude Code Commands
This project includes Claude Code slash commands for common tasks:
- `/test` - Run all tests
- `/build` - Build the binary
- `/lint` - Run linters
- `/coverage` - Generate coverage report
## Questions or Issues?
- Open an issue for bugs or feature requests
- Start a discussion for questions or ideas
- Check existing issues before creating new ones
Do not report vulnerabilities in public issues. See [SECURITY.md](SECURITY.md).
## License
By contributing to RelSpec, you agree that your contributions will be licensed under the Apache License 2.0.
Contributions are licensed under the Apache License 2.0.
+32 -12
View File
@@ -1,4 +1,4 @@
.PHONY: all build test test-unit test-integration lint coverage clean install help docker-up docker-down docker-test docker-test-integration start stop release release-version godoc vet fmt fmt-check staticcheck govulncheck check
.PHONY: all build test test-unit test-integration lint coverage clean install help docker-up docker-down docker-test docker-test-integration start stop release release-version rerelease godoc vet fmt fmt-check staticcheck govulncheck check
# Binary name
BINARY_NAME=relspec
@@ -20,9 +20,11 @@ STATICCHECK = go run honnef.co/go/tools/cmd/staticcheck@latest
GOVULNCHECK = go run golang.org/x/vuln/cmd/govulncheck@latest
# Version information
VERSION := $(shell git describe --tags --always --dirty 2>/dev/null || echo "dev")
# VERSION file is the single source of truth (bumped by `make release-version`).
# Falls back to git describe for ad-hoc builds where VERSION hasn't been committed yet.
VERSION := $(shell [ -f VERSION ] && echo "v$$(cat VERSION | tr -d '[:space:]')" || git describe --tags --always --dirty 2>/dev/null || echo "dev")
BUILD_DATE := $(shell date -u +"%Y-%m-%d %H:%M:%S UTC")
LDFLAGS := -X 'main.version=$(VERSION)' -X 'main.buildDate=$(BUILD_DATE)'
LDFLAGS := -X 'git.warky.dev/wdevs/relspecgo/pkg/buildinfo.Version=$(VERSION)' -X 'git.warky.dev/wdevs/relspecgo/pkg/buildinfo.BuildDate=$(BUILD_DATE)'
# Auto-detect container runtime (Docker or Podman)
CONTAINER_RUNTIME := $(shell \
@@ -207,7 +209,7 @@ docker-test-integration: docker-up ## Start DB and run integration tests
$(GOTEST) -v ./pkg/readers/pgsql/ -count=1 || (make docker-down && exit 1)
@make docker-down
release: ## Create and push a new release tag (auto-increments patch version)
release: lint fmt-check test build ## Run lint, format check, tests, build, then create and push a new release tag
@echo "Creating new release..."
@latest_tag=$$(git describe --tags --abbrev=0 2>/dev/null || echo ""); \
if [ -z "$$latest_tag" ]; then \
@@ -223,6 +225,15 @@ release: ## Create and push a new release tag (auto-increments patch version)
echo "Creating new release: $$version"; \
commit_logs=$$(git log "$${latest_tag}..HEAD" --pretty=format:"- %s" --no-merges); \
fi; \
PKGVER=$${version#v}; \
echo "Updating package metadata to $$PKGVER..."; \
echo "$$PKGVER" > VERSION; \
sed -i "s/^pkgver=.*/pkgver=$$PKGVER/" linux/arch/PKGBUILD; \
sed -i "s/^Version:.*/Version: $$PKGVER/" linux/centos/relspec.spec; \
git add VERSION linux/arch/PKGBUILD linux/centos/relspec.spec; \
if ! git diff --cached --quiet; then \
git commit -m "chore(release): update package version to $$PKGVER"; \
fi; \
if [ -z "$$commit_logs" ]; then \
tag_message="Release $$version"; \
else \
@@ -232,21 +243,30 @@ release: ## Create and push a new release tag (auto-increments patch version)
git push origin "$$version"; \
echo "Tag $$version created and pushed to remote repository."
release-version: ## Auto-increment patch version, update package files, commit, tag, and push
@CURRENT=$$(git describe --tags --abbrev=0 2>/dev/null || echo "v0.0.0"); \
MAJOR=$$(echo $$CURRENT | sed 's/v\([0-9]*\)\.\([0-9]*\)\.\([0-9]*\).*/\1/'); \
MINOR=$$(echo $$CURRENT | sed 's/v\([0-9]*\)\.\([0-9]*\)\.\([0-9]*\).*/\2/'); \
PATCH=$$(echo $$CURRENT | sed 's/v\([0-9]*\)\.\([0-9]*\)\.\([0-9]*\).*/\3/'); \
NEXT="v$$MAJOR.$$MINOR.$$((PATCH + 1))"; \
release-version: lint fmt-check ## Run lint and format check, then auto-increment patch version, update VERSION/package files, commit, tag, and push
@CURRENT=$$([ -f VERSION ] && cat VERSION | tr -d '[:space:]' || (git describe --tags --abbrev=0 2>/dev/null | sed 's/^v//') || echo "0.0.0"); \
MAJOR=$$(echo $$CURRENT | cut -d. -f1); \
MINOR=$$(echo $$CURRENT | cut -d. -f2); \
PATCH=$$(echo $$CURRENT | cut -d. -f3); \
PKGVER="$$MAJOR.$$MINOR.$$((PATCH + 1))"; \
echo "Current: $$CURRENT → Next: $$NEXT"; \
NEXT="v$$PKGVER"; \
echo "Current: v$$CURRENT → Next: $$NEXT"; \
echo "$$PKGVER" > VERSION; \
sed -i "s/^pkgver=.*/pkgver=$$PKGVER/" linux/arch/PKGBUILD; \
sed -i "s/^Version:.*/Version: $$PKGVER/" linux/centos/relspec.spec; \
git add linux/arch/PKGBUILD linux/centos/relspec.spec; \
git add VERSION linux/arch/PKGBUILD linux/centos/relspec.spec; \
git commit -m "chore(release): update package version to $$PKGVER"; \
git tag -a "$$NEXT" -m "Release $$NEXT"; \
git push origin HEAD "$$NEXT"; \
echo "Pushed $$NEXT — release workflow triggered"
rerelease: lint fmt-check ## Move the latest tag to HEAD and force push it
@TAG=$$(git describe --tags --abbrev=0 2>/dev/null); \
if [ -z "$$TAG" ]; then echo "No existing tags found"; exit 1; fi; \
echo "Moving $$TAG to $$(git rev-parse --short HEAD)"; \
git tag -f -a "$$TAG" -m "Release $$TAG" HEAD; \
git push --force origin "$$TAG"; \
echo "Pushed $$TAG — release workflow triggered"
help: ## Display this help screen
@grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-20s\033[0m %s\n", $$1, $$2}'
+6 -2
View File
@@ -23,6 +23,8 @@ go install -v git.warky.dev/wdevs/relspecgo/cmd/relspec@latest
| **Readers** | `bun` `dbml` `dctx` `drawdb` `drizzle` `gorm` `graphql` `json` `mssql` `pgsql` `prisma` `sqldir` `sqlite` `typeorm` `yaml` |
| **Writers** | `bun` `dbml` `dctx` `drawdb` `drizzle` `gorm` `graphql` `json` `mssql` `pgsql` `prisma` `sqlexec` `sqlite` `template` `typeorm` `yaml` |
See [docs/FORMAT_EXAMPLES.md](docs/FORMAT_EXAMPLES.md) for usage examples covering every format.
## Commands
### `convert` — Schema conversion
@@ -126,7 +128,7 @@ relspec job run build-schema
version: 1
jobs:
build-schema:
command: convert # closed allow-list: convert | merge | scripts-list
command: convert # closed allow-list: convert | merge | split | scripts-list | scripts-exec | templ | inspect | diff
description: Merge the DBML sources and emit PostgreSQL DDL
inputs:
- path: schema/core.dbml
@@ -147,7 +149,9 @@ resolved relative to the job file and may not escape it, and remote database
credentials are referenced by environment-variable name (`conn_env:`) and
redacted from logs. The whole plan — unknown commands/formats, duplicate job
names, missing inputs, path traversal, dependency cycles — is validated before
any job runs. See [docs/JOB_FILES.md](docs/JOB_FILES.md).
any job runs. Path-like fields also support `${NAME}` environment-variable
references, which are expanded and safety-checked during pre-flight. See
[docs/JOB_FILES.md](docs/JOB_FILES.md).
### `edit` — Interactive TUI editor
+12
View File
@@ -0,0 +1,12 @@
# Security Policy
## Supported versions
Security fixes are released for the latest minor version of RelSpec.
## Reporting a vulnerability
Please do not open a public issue for security problems.
Report privately by email: warkydevs@gmail.com
Reports are reviewed on a best-effort basis. No response time or fix timeline is
guaranteed. Reporters may be credited in the release notes if they wish.
+20 -9
View File
@@ -4,20 +4,25 @@
- [✔️] **Database Inspector**
- [✔️] PostgreSQL driver (reader + writer)
- [ ] MySQL driver
- [ ] MySQL driver (only MariaDB datatype conversion in pkg/mariadb, no reader/writer)
- [✔️] SQLite driver (reader + writer with automatic schema flattening)
- [ ] MSSQL driver
- [✔️] MSSQL driver (reader + writer, generated and identity columns)
- [✔️] Foreign key detection
- [✔️] Index extraction
- [✔️] .sql file generation (PostgreSQL, SQLite)
- [✔️] .dbml: Database Markup Language (DBML) for textual schema representation.
- [✔️] Prisma schema support (PSL format) .prisma
- [ ] Sequelize (Typescript/Javascript) (Use templates, 💲 Someone can do this, not me)
- [✔️] Drizzle ORM support .ts (TypeScript / JavaScript) (Mr. Edd wanted to move from Prisma to Drizzle. If you are bugs, you are welcome to do pull requests or issues)
- [☠️] Entity Framework (.NET) model .edmx (Fuck no, EDMX files were bloated, verbose XML nightmares—hard to merge, error-prone, and a pain in teams. Microsoft wisely ditched them in EF Core for code-first. Classic overkill from old MS era.)
- [✔️] TypeORM support
- [] .hbm.xml / schema.xml: Hibernate/Propel mappings (Java/PHP) (💲 Someone can do this, not me)
- [ ] Django models.py (Python classes), Sequelize migrations (JS) (💲 Someone can do this, not me)
- [] .avsc: Avro schema (JSON format for data serialization) (💲 Someone can do this, not me)
- [ ] .hbm.xml / schema.xml: Hibernate/Propel mappings (Java/PHP) (Use templates, 💲 Someone can do this, not me)
- [ ] Django models.py (Python classes), Sequelize migrations (JS) (Use templates, 💲 Someone can do this, not me)
- [ ] SQLAlchemy, Python (Use templates, 💲 Someone can do this, not me)
- [ ] .avsc: Avro schema (JSON format for data serialization) (Use templates, 💲 Someone can do this, not me)
- [ ] Rails schema in db/schema.rb (Use templates, 💲 Someone can do this, not me)
- [ ] Laravel migration (Use templates, 💲 Someone can do this, not me)
- [ ] Hibernate / JPA (Use templates, 💲 Someone can do this, not me)
- [✔️] GraphQL schema generation
## UI
@@ -39,13 +44,19 @@
## Advanced Features
- [ ] Dry-run mode for validation
- [x] Diff tool for comparing specifications
- [ ] Migration script generation
- [ ] Dry-run mode for validation (only `job run --dry-run`; not on convert/merge/split)
- [✔️] Diff tool for comparing specifications
- [✔️] Migration script generation (PostgreSQL diff, live database by default for direct output)
- [ ] Custom type mapping configuration
- [ ] Batch processing support
- [ ] Batch processing support (partial: job files run named workflows via `relspec job run`)
- [ ] Watch mode for auto-regeneration
## Distribution
- [ ] NSIS Windows installer
- [ ] Check https://git.warky.dev/wdevs/relspecgo/releases for new releases and prompt to update
- [ ] CI pipeline to build and publish the Windows installer
## Future Considerations
- [ ] Web UI for visual editing
+1
View File
@@ -0,0 +1 @@
1.0.85
+136 -14
View File
@@ -3,6 +3,7 @@ package main
import (
stdjson "encoding/json"
"fmt"
"io"
"os"
"strings"
"time"
@@ -21,6 +22,7 @@ import (
"git.warky.dev/wdevs/relspecgo/pkg/readers/graphql"
"git.warky.dev/wdevs/relspecgo/pkg/readers/json"
"git.warky.dev/wdevs/relspecgo/pkg/readers/mssql"
"git.warky.dev/wdevs/relspecgo/pkg/readers/mysql"
"git.warky.dev/wdevs/relspecgo/pkg/readers/pgsql"
"git.warky.dev/wdevs/relspecgo/pkg/readers/prisma"
"git.warky.dev/wdevs/relspecgo/pkg/readers/sqlite"
@@ -36,6 +38,7 @@ import (
wgraphql "git.warky.dev/wdevs/relspecgo/pkg/writers/graphql"
wjson "git.warky.dev/wdevs/relspecgo/pkg/writers/json"
wmssql "git.warky.dev/wdevs/relspecgo/pkg/writers/mssql"
wmysql "git.warky.dev/wdevs/relspecgo/pkg/writers/mysql"
wpgsql "git.warky.dev/wdevs/relspecgo/pkg/writers/pgsql"
wprisma "git.warky.dev/wdevs/relspecgo/pkg/writers/prisma"
wsqlite "git.warky.dev/wdevs/relspecgo/pkg/writers/sqlite"
@@ -57,6 +60,7 @@ var (
convertNullableArrays string
convertContinueOnError bool
convertExtraFields string
convertDryRun bool
)
var convertCmd = &cobra.Command{
@@ -185,6 +189,8 @@ func init() {
convertCmd.Flags().BoolVar(&convertContinueOnError, "continue-on-error", false, "Prepend \\set ON_ERROR_STOP off to generated SQL so psql continues past errors (pgsql output only)")
convertCmd.Flags().StringVar(&convertExtraFields, "extra-fields", "", "Path to JSON file containing extra Bun model fields to inject (bun output only); fields support target_table, name, type, bun_tag, json_tag, comment")
convertCmd.Flags().BoolVar(&convertDryRun, "dry-run", false, "Read and validate the input and print the plan without writing any output")
err := convertCmd.MarkFlagRequired("from")
if err != nil {
fmt.Fprintf(os.Stderr, "Error marking from flag as required: %v\n", err)
@@ -229,6 +235,7 @@ func runConvert(cmd *cobra.Command, args []string) error {
if err != nil {
return fmt.Errorf("failed to read source: %w", err)
}
finalizeCommentedRefs(db, stderrWarn)
fmt.Fprintf(os.Stderr, " ✓ Successfully read database '%s'\n", db.Name)
fmt.Fprintf(os.Stderr, " Found: %d schema(s)\n", len(db.Schemas))
@@ -239,6 +246,22 @@ func runConvert(cmd *cobra.Command, args []string) error {
}
fmt.Fprintf(os.Stderr, " Found: %d table(s)\n\n", totalTables)
if convertDryRun {
if err := validateWriteTarget(db, convertTargetType, convertPackageName, convertSchemaFilter, convertExtraFields); err != nil {
return fmt.Errorf("dry run validation failed: %w", err)
}
out := outWriter(cmd)
fmt.Fprintf(out, "RelSpec convert plan (dry run - nothing written):\n")
fmt.Fprintf(out, " Input: %s database '%s'\n", convertSourceType, db.Name)
fmt.Fprintf(out, " Output: %s -> %s\n", convertTargetType, convertTargetPath)
if convertSchemaFilter != "" {
fmt.Fprintf(out, " Schema filter: %s\n", convertSchemaFilter)
}
printDryRunPlan(out, db)
fmt.Fprintf(os.Stderr, "=== Dry run complete: no output written ===\n\n")
return nil
}
// Write to target format
fmt.Fprintf(os.Stderr, "[2/2] Writing to target format...\n")
fmt.Fprintf(os.Stderr, " Format: %s\n", convertTargetType)
@@ -261,6 +284,18 @@ func runConvert(cmd *cobra.Command, args []string) error {
return nil
}
// finalizeCommentedRefs resolves DBML `// Ref:` comments against the fully
// loaded model and warns about refs whose target is not loaded.
func finalizeCommentedRefs(db *models.Database, warn func(string)) {
for _, w := range dbml.ResolveCommentedRefs(db, true) {
warn(w)
}
}
func stderrWarn(msg string) {
fmt.Fprintf(os.Stderr, " ⚠ %s\n", msg)
}
func readDatabaseListForConvert(dbType string, files []string) (*models.Database, error) {
if len(files) == 0 {
return nil, fmt.Errorf("file list is empty")
@@ -367,6 +402,12 @@ func readDatabaseForConvert(dbType, filePath, connString string) (*models.Databa
}
reader = mssql.NewReader(newReaderOptions("", connString))
case "mysql", "mariadb":
if connString == "" {
return nil, fmt.Errorf("connection string is required for MySQL format")
}
reader = mysql.NewReader(newReaderOptions("", connString))
case "sqlite", "sqlite3":
// SQLite can use either file path or connection string
dbPath := filePath
@@ -395,23 +436,12 @@ func writeDatabase(db *models.Database, dbType, outputPath, packageName, schemaF
writerOpts := newWriterOptions(outputPath, packageName, flattenSchema, nullableTypes, nullableArrays, continueOnError)
if extraFields != "" {
if !strings.EqualFold(dbType, "bun") {
return fmt.Errorf("--extra-fields is only supported for Bun output")
}
extraFieldsJSON, err := os.ReadFile(extraFields)
extraFieldsJSON, err := loadExtraFields(dbType, extraFields)
if err != nil {
return fmt.Errorf("failed to read --extra-fields file %q: %w", extraFields, err)
}
var parsed []wbun.ExtraFieldConfig
if err := stdjson.Unmarshal(extraFieldsJSON, &parsed); err != nil {
return fmt.Errorf("invalid --extra-fields JSON in %q: %w", extraFields, err)
}
if len(parsed) == 0 {
return fmt.Errorf("--extra-fields must contain at least one field")
return err
}
writerOpts.Metadata = map[string]interface{}{
"extra_fields": string(extraFieldsJSON),
"extra_fields": extraFieldsJSON,
}
}
@@ -452,6 +482,9 @@ func writeDatabase(db *models.Database, dbType, outputPath, packageName, schemaF
case "mssql", "sqlserver", "mssql2016", "mssql2017", "mssql2019", "mssql2022":
writer = wmssql.NewWriter(writerOpts)
case "mysql", "mariadb":
writer = wmysql.NewWriter(writerOpts)
case "sqlite", "sqlite3":
writer = wsqlite.NewWriter(writerOpts)
@@ -516,6 +549,95 @@ func writeDatabase(db *models.Database, dbType, outputPath, packageName, schemaF
return nil
}
// loadExtraFields reads and validates the --extra-fields JSON file, returning
// its raw content.
func loadExtraFields(dbType, path string) (string, error) {
if !strings.EqualFold(dbType, "bun") {
return "", fmt.Errorf("--extra-fields is only supported for Bun output")
}
extraFieldsJSON, err := os.ReadFile(path)
if err != nil {
return "", fmt.Errorf("failed to read --extra-fields file %q: %w", path, err)
}
var parsed []wbun.ExtraFieldConfig
if err := stdjson.Unmarshal(extraFieldsJSON, &parsed); err != nil {
return "", fmt.Errorf("invalid --extra-fields JSON in %q: %w", path, err)
}
if len(parsed) == 0 {
return "", fmt.Errorf("--extra-fields must contain at least one field")
}
return string(extraFieldsJSON), nil
}
// validateWriteTarget performs the checks writeDatabase makes before writing,
// without constructing a writer or touching the output path. Used by --dry-run.
func validateWriteTarget(db *models.Database, dbType, packageName, schemaFilter, extraFields string) error {
if extraFields != "" {
if _, err := loadExtraFields(dbType, extraFields); err != nil {
return err
}
}
switch strings.ToLower(dbType) {
case "dbml", "dctx", "drawdb", "json", "yaml", "yml", "drizzle",
"pgsql", "postgres", "postgresql", "sql",
"mssql", "sqlserver", "mssql2016", "mssql2017", "mssql2019", "mssql2022",
"sqlite", "sqlite3", "prisma", "typeorm", "graphql", "gql":
case "gorm", "bun":
if packageName == "" {
return fmt.Errorf("package name is required for %s format (use --package flag)", dbType)
}
default:
return fmt.Errorf("unsupported target format: %s", dbType)
}
if schemaFilter != "" {
for _, schema := range db.Schemas {
if schema.Name == schemaFilter {
return nil
}
}
return fmt.Errorf("schema '%s' not found in database. Available schemas: %v",
schemaFilter, getSchemaNames(db))
}
if strings.EqualFold(dbType, "dctx") {
if len(db.Schemas) == 0 {
return fmt.Errorf("no schemas found in database")
}
if len(db.Schemas) > 1 {
return fmt.Errorf("multiple schemas found, please specify which schema to export using --schema flag. Available schemas: %v",
getSchemaNames(db))
}
}
return nil
}
// outWriter returns the command's stdout, falling back to os.Stdout when the
// command is nil (tests call the run functions directly).
func outWriter(cmd *cobra.Command) io.Writer {
if cmd == nil {
return os.Stdout
}
return cmd.OutOrStdout()
}
// printDryRunPlan prints the schemas and tables that would be written.
func printDryRunPlan(out io.Writer, db *models.Database) {
for _, schema := range db.Schemas {
names := make([]string, 0, len(schema.Tables))
for _, t := range schema.Tables {
names = append(names, t.Name)
}
fmt.Fprintf(out, " schema %q: %d table(s)", schema.Name, len(schema.Tables))
if len(names) > 0 {
fmt.Fprintf(out, " [%s]", strings.Join(names, ", "))
}
fmt.Fprintln(out)
}
}
// getSchemaNames returns a slice of schema names from a database
func getSchemaNames(db *models.Database) []string {
names := make([]string, len(db.Schemas))
+8 -8
View File
@@ -229,43 +229,43 @@ func readDatabase(dbType, filePath, connString, label string) (*models.Database,
if filePath == "" {
return nil, fmt.Errorf("%s: file path is required for DBML format", label)
}
reader = dbml.NewReader(&readers.ReaderOptions{FilePath: filePath})
reader = dbml.NewReader(newReaderOptions(filePath, ""))
case "dctx":
if filePath == "" {
return nil, fmt.Errorf("%s: file path is required for DCTX format", label)
}
reader = dctx.NewReader(&readers.ReaderOptions{FilePath: filePath})
reader = dctx.NewReader(newReaderOptions(filePath, ""))
case "drawdb":
if filePath == "" {
return nil, fmt.Errorf("%s: file path is required for DrawDB format", label)
}
reader = drawdb.NewReader(&readers.ReaderOptions{FilePath: filePath})
reader = drawdb.NewReader(newReaderOptions(filePath, ""))
case "json":
if filePath == "" {
return nil, fmt.Errorf("%s: file path is required for JSON format", label)
}
reader = json.NewReader(&readers.ReaderOptions{FilePath: filePath})
reader = json.NewReader(newReaderOptions(filePath, ""))
case "yaml":
if filePath == "" {
return nil, fmt.Errorf("%s: file path is required for YAML format", label)
}
reader = yaml.NewReader(&readers.ReaderOptions{FilePath: filePath})
reader = yaml.NewReader(newReaderOptions(filePath, ""))
case "sqldir", "scripts", "scriptdir":
if filePath == "" {
return nil, fmt.Errorf("%s: file path is required for SQL directory format", label)
}
reader = sqldir.NewReader(&readers.ReaderOptions{FilePath: filePath})
reader = sqldir.NewReader(newReaderOptions(filePath, ""))
case "pgsql", "postgres", "postgresql":
if connString == "" {
return nil, fmt.Errorf("%s: connection string is required for PostgreSQL format", label)
}
reader = pgsql.NewReader(&readers.ReaderOptions{ConnectionString: connString})
reader = pgsql.NewReader(newReaderOptions("", connString))
case "sqlite", "sqlite3":
// SQLite can use either file path or connection string
@@ -276,7 +276,7 @@ func readDatabase(dbType, filePath, connString, label string) (*models.Database,
if dbPath == "" {
return nil, fmt.Errorf("%s: file path or connection string is required for SQLite format", label)
}
reader = sqlite.NewReader(&readers.ReaderOptions{FilePath: dbPath})
reader = sqlite.NewReader(newReaderOptions(dbPath, ""))
default:
return nil, fmt.Errorf("%s: unsupported database format: %s", label, dbType)
+158
View File
@@ -0,0 +1,158 @@
package main
import (
"bytes"
"os"
"path/filepath"
"strings"
"testing"
"github.com/spf13/cobra"
)
func newDryRunCmd() (*cobra.Command, *bytes.Buffer) {
var buf bytes.Buffer
cmd := &cobra.Command{}
cmd.SetOut(&buf)
return cmd, &buf
}
func TestRunConvert_DryRunWritesNothing(t *testing.T) {
defer func(a, b, c, d string, e bool) {
convertSourceType, convertSourcePath, convertTargetType, convertTargetPath, convertDryRun = a, b, c, d, e
}(convertSourceType, convertSourcePath, convertTargetType, convertTargetPath, convertDryRun)
dir := t.TempDir()
in := filepath.Join(dir, "in.json")
out := filepath.Join(dir, "out.json")
writeTestJSON(t, in, []string{"users", "posts"})
convertSourceType, convertSourcePath = "json", in
convertTargetType, convertTargetPath = "json", out
convertDryRun = true
cmd, buf := newDryRunCmd()
if err := runConvert(cmd, nil); err != nil {
t.Fatalf("dry run: %v", err)
}
if _, err := os.Stat(out); !os.IsNotExist(err) {
t.Fatal("dry run must not create the output file")
}
for _, want := range []string{"dry run", "users", "posts", out} {
if !strings.Contains(buf.String(), want) {
t.Errorf("plan missing %q:\n%s", want, buf.String())
}
}
// Normal behavior is unchanged.
convertDryRun = false
if err := runConvert(cmd, nil); err != nil {
t.Fatalf("real run: %v", err)
}
if _, err := os.Stat(out); err != nil {
t.Fatalf("real run should write output: %v", err)
}
}
func TestRunConvert_DryRunValidatesTarget(t *testing.T) {
defer func(a, b, c, d string, e bool) {
convertSourceType, convertSourcePath, convertTargetType, convertTargetPath, convertDryRun = a, b, c, d, e
}(convertSourceType, convertSourcePath, convertTargetType, convertTargetPath, convertDryRun)
dir := t.TempDir()
in := filepath.Join(dir, "in.json")
out := filepath.Join(dir, "models")
writeTestJSON(t, in, []string{"users"})
convertSourceType, convertSourcePath = "json", in
convertDryRun = true
// gorm without --package must fail validation, as a real run would.
convertTargetType, convertTargetPath = "gorm", out
cmd, _ := newDryRunCmd()
err := runConvert(cmd, nil)
if err == nil || !strings.Contains(err.Error(), "package name is required") {
t.Fatalf("expected package validation error, got %v", err)
}
convertTargetType = "nope"
err = runConvert(cmd, nil)
if err == nil || !strings.Contains(err.Error(), "unsupported target format") {
t.Fatalf("expected unsupported format error, got %v", err)
}
if _, statErr := os.Stat(out); !os.IsNotExist(statErr) {
t.Fatal("dry run must not create the output path")
}
}
func TestRunSplit_DryRunWritesNothing(t *testing.T) {
defer func(a, b, c, d, e string, f bool) {
splitSourceType, splitSourcePath, splitTargetType, splitTargetPath, splitTables, splitDryRun = a, b, c, d, e, f
}(splitSourceType, splitSourcePath, splitTargetType, splitTargetPath, splitTables, splitDryRun)
dir := t.TempDir()
in := filepath.Join(dir, "in.json")
out := filepath.Join(dir, "subset.json")
writeTestJSON(t, in, []string{"users", "posts", "comments"})
splitSourceType, splitSourcePath = "json", in
splitTargetType, splitTargetPath = "json", out
splitTables = "users,posts"
splitDryRun = true
cmd, buf := newDryRunCmd()
if err := runSplit(cmd, nil); err != nil {
t.Fatalf("dry run: %v", err)
}
if _, err := os.Stat(out); !os.IsNotExist(err) {
t.Fatal("dry run must not create the output file")
}
got := buf.String()
if !strings.Contains(got, "2 table(s)") || strings.Contains(got, "comments") {
t.Errorf("plan should show only the 2 selected tables:\n%s", got)
}
// A selection that matches nothing fails validation in dry-run too.
splitTables = "does_not_exist"
if err := runSplit(cmd, nil); err == nil {
t.Fatal("expected error for empty selection")
}
}
func TestRunMerge_DryRunWritesNothing(t *testing.T) {
saved := saveMergeState()
defer restoreMergeState(saved)
defer func(v bool) { mergeDryRun = v }(mergeDryRun)
dir := t.TempDir()
target := filepath.Join(dir, "target.json")
source := filepath.Join(dir, "source.json")
out := filepath.Join(dir, "merged.json")
writeTestJSON(t, target, []string{"users"})
writeTestJSON(t, source, []string{"posts"})
mergeTargetType, mergeTargetPath, mergeTargetConn = "json", target, ""
mergeSourceType, mergeSourcePath, mergeSourceConn = "json", source, ""
mergeFromList = nil
mergeOutputType, mergeOutputPath, mergeOutputConn = "json", out, ""
mergeSkipTables, mergeReportPath = "", ""
mergeDryRun = true
cmd, buf := newDryRunCmd()
if err := runMerge(cmd, nil); err != nil {
t.Fatalf("dry run: %v", err)
}
if _, err := os.Stat(out); !os.IsNotExist(err) {
t.Fatal("dry run must not create the output file")
}
for _, want := range []string{"dry run", "users", "posts", out} {
if !strings.Contains(buf.String(), want) {
t.Errorf("plan missing %q:\n%s", want, buf.String())
}
}
mergeOutputType = "nope"
if err := runMerge(cmd, nil); err == nil || !strings.Contains(err.Error(), "unsupported format") {
t.Fatalf("expected unsupported output format error, got %v", err)
}
}
+369 -23
View File
@@ -1,6 +1,7 @@
package main
import (
"bytes"
"fmt"
"io"
"os"
@@ -11,6 +12,8 @@ import (
"github.com/spf13/cobra"
"git.warky.dev/wdevs/relspecgo/pkg/diff"
"git.warky.dev/wdevs/relspecgo/pkg/inspector"
"git.warky.dev/wdevs/relspecgo/pkg/jobs"
"git.warky.dev/wdevs/relspecgo/pkg/merge"
"git.warky.dev/wdevs/relspecgo/pkg/models"
@@ -18,6 +21,7 @@ import (
"git.warky.dev/wdevs/relspecgo/pkg/readers/sqldir"
"git.warky.dev/wdevs/relspecgo/pkg/writers"
wpgsql "git.warky.dev/wdevs/relspecgo/pkg/writers/pgsql"
"git.warky.dev/wdevs/relspecgo/pkg/writers/sqlexec"
wtemplate "git.warky.dev/wdevs/relspecgo/pkg/writers/template"
)
@@ -56,8 +60,9 @@ inputs, output and options:
logfile: .relspec/log/build-schema.log
Rules and guarantees:
- command is a closed allow-list (convert, merge, scripts-list, templ). Arbitrary
shell strings are never executed.
- command is a closed allow-list (convert, merge, split, scripts-list,
scripts-exec, templ, inspect, diff). Arbitrary shell strings are never
executed.
- Every path is relative to the directory holding the job file and may not
escape it. Absolute and home-relative paths are rejected.
- Remote database credentials are referenced by environment-variable name
@@ -122,6 +127,9 @@ func loadJobSet() (*jobs.Set, error) {
if err := set.Validate(); err != nil {
return nil, err
}
for _, w := range set.Warnings {
fmt.Fprintf(os.Stderr, "warning: %s\n", w)
}
return set, nil
}
@@ -180,12 +188,14 @@ func executeJobPlan(set *jobs.Set, name string, dryRun, noDeps bool, out io.Writ
// Pre-flight: resolve and check paths, output policy and env vars for the
// whole plan before anything runs. A failure here means no job executes.
resolved := make([]*resolvedJob, len(plan))
byName := make(map[string]*resolvedJob, len(plan))
for i, j := range plan {
rj, perr := preflightJob(j)
rj, perr := preflightJob(j, byName)
if perr != nil {
return fmt.Errorf("job %q: %w", j.Name, perr)
}
resolved[i] = rj
byName[j.Name] = rj
}
if dryRun {
@@ -217,7 +227,12 @@ type resolvedJob struct {
outputConn string // resolved connection string (secret)
outputConnEnv string
logPath string
logPolicy jobs.LogPolicy
templatePath string
reportPath string // "" for a diff summary written to the log
reportFormat string
rulesPath string // "" means inspector defaults
selection *splitSelection
secrets []string // resolved secret values to redact from logs
}
@@ -226,33 +241,63 @@ type resolvedInput struct {
path string // "" when the input is a database
conn string // resolved connection string (secret)
connEnv string
fromJob string // producer job name when this input came from from_job
}
func preflightJob(j *jobs.Job) (*resolvedJob, error) {
func preflightJob(j *jobs.Job, resolvedByName map[string]*resolvedJob) (*resolvedJob, error) {
root := j.Dir()
rj := &resolvedJob{job: j, root: root}
rj := &resolvedJob{job: j, root: root, logPolicy: j.ResolvedLogPolicy()}
expandPath := func(label, value string) (string, error) {
expanded, err := jobs.ExpandEnv(value)
if err != nil {
return "", fmt.Errorf("%s: %w", label, err)
}
return expanded, nil
}
if j.Logfile != "" {
p, err := jobs.SafeJoin(root, j.Logfile)
logfile, err := expandPath("logfile", j.Logfile)
if err != nil {
return nil, err
}
p, err := jobs.SafeJoin(root, logfile)
if err != nil {
return nil, fmt.Errorf("logfile: %w", err)
}
rj.logPath = p
}
if j.Template != "" {
p, err := jobs.SafeJoin(root, j.Template)
templatePath, err := expandPath("template", j.Template)
if err != nil {
return nil, err
}
p, err := jobs.SafeJoin(root, templatePath)
if err != nil {
return nil, fmt.Errorf("template: %w", err)
}
info, err := os.Stat(p)
if err != nil || info.IsDir() {
return nil, fmt.Errorf("template %q: not found or is a directory", j.Template)
return nil, fmt.Errorf("template %q: not found or is a directory", templatePath)
}
rj.templatePath = p
}
for i, in := range j.Inputs {
ri := resolvedInput{format: strings.ToLower(in.Format)}
if in.FromJob != "" {
producer, ok := resolvedByName[in.FromJob]
if !ok {
return nil, fmt.Errorf("input[%d]: from_job %q is not in this plan (do not use --no-deps with from_job inputs)", i, in.FromJob)
}
if producer.outputPath == "" {
return nil, fmt.Errorf("input[%d]: from_job %q does not write a file output", i, in.FromJob)
}
ri.path = producer.outputPath
ri.format = strings.ToLower(producer.job.Output.Format)
ri.fromJob = in.FromJob
rj.inputs = append(rj.inputs, ri)
continue
}
if in.ConnEnv != "" {
v, ok := os.LookupEnv(in.ConnEnv)
if !ok || v == "" {
@@ -262,16 +307,20 @@ func preflightJob(j *jobs.Job) (*resolvedJob, error) {
ri.connEnv = in.ConnEnv
rj.secrets = append(rj.secrets, v)
} else {
p, err := jobs.SafeJoin(root, in.Path)
inputPath, err := expandPath(fmt.Sprintf("input[%d]", i), in.Path)
if err != nil {
return nil, err
}
p, err := jobs.SafeJoin(root, inputPath)
if err != nil {
return nil, fmt.Errorf("input[%d]: %w", i, err)
}
info, err := os.Stat(p)
if err != nil {
return nil, fmt.Errorf("input[%d]: %s: file not found", i, in.Path)
return nil, fmt.Errorf("input[%d]: %s: file not found", i, inputPath)
}
if info.IsDir() {
return nil, fmt.Errorf("input[%d]: %s: is a directory, not a file", i, in.Path)
return nil, fmt.Errorf("input[%d]: %s: is a directory, not a file", i, inputPath)
}
ri.path = p
}
@@ -279,13 +328,17 @@ func preflightJob(j *jobs.Job) (*resolvedJob, error) {
}
for _, d := range j.ScriptDirs {
p, err := jobs.SafeJoin(root, d)
scriptDir, err := expandPath("script_dir", d)
if err != nil {
return nil, err
}
p, err := jobs.SafeJoin(root, scriptDir)
if err != nil {
return nil, fmt.Errorf("script_dir %q: %w", d, err)
}
info, err := os.Stat(p)
if err != nil {
return nil, fmt.Errorf("script_dir %q: not found", d)
return nil, fmt.Errorf("script_dir %q: not found", scriptDir)
}
if !info.IsDir() {
return nil, fmt.Errorf("script_dir %q: not a directory", d)
@@ -303,16 +356,65 @@ func preflightJob(j *jobs.Job) (*resolvedJob, error) {
rj.outputConnEnv = j.Output.ConnEnv
rj.secrets = append(rj.secrets, v)
} else {
p, err := jobs.SafeJoin(root, j.Output.Path)
outputPath, err := expandPath("output", j.Output.Path)
if err != nil {
return nil, err
}
p, err := jobs.SafeJoin(root, outputPath)
if err != nil {
return nil, fmt.Errorf("output: %w", err)
}
if _, err := os.Stat(p); err == nil && !j.Output.Overwrite {
return nil, fmt.Errorf("output %s already exists (set output.overwrite: true to replace it)", j.Output.Path)
return nil, fmt.Errorf("output %s already exists (set output.overwrite: true to replace it)", outputPath)
}
rj.outputPath = p
}
}
if j.Rules != "" {
rulesPath, err := expandPath("rules", j.Rules)
if err != nil {
return nil, err
}
p, err := jobs.SafeJoin(root, rulesPath)
if err != nil {
return nil, fmt.Errorf("rules: %w", err)
}
info, err := os.Stat(p)
if err != nil || info.IsDir() {
return nil, fmt.Errorf("rules %q: not found or is a directory", rulesPath)
}
rj.rulesPath = p
}
if j.Report != nil {
rj.reportFormat = strings.ToLower(j.Report.Format)
if j.Report.Path != "" {
reportPath, err := expandPath("report", j.Report.Path)
if err != nil {
return nil, err
}
p, err := jobs.SafeJoin(root, reportPath)
if err != nil {
return nil, fmt.Errorf("report: %w", err)
}
if _, err := os.Stat(p); err == nil && !j.Report.Overwrite {
return nil, fmt.Errorf("report %s already exists (set report.overwrite: true to replace it)", reportPath)
}
rj.reportPath = p
}
}
if j.Select != nil {
rj.selection = &splitSelection{
Schemas: j.Select.Schemas,
Tables: j.Select.Tables,
ExcludeSchemas: j.Select.ExcludeSchemas,
ExcludeTables: j.Select.ExcludeTables,
DatabaseName: j.Select.DatabaseName,
}
}
return rj, nil
}
@@ -325,9 +427,12 @@ func printResolvedJob(out io.Writer, n, total int, rj *resolvedJob) {
}
fmt.Fprintf(out, " job file: %s\n", j.SourceFile)
for _, ri := range rj.inputs {
if ri.path != "" {
switch {
case ri.fromJob != "":
fmt.Fprintf(out, " input: %s (%s) from job %q\n", ri.path, ri.format, ri.fromJob)
case ri.path != "":
fmt.Fprintf(out, " input: %s (%s)\n", ri.path, ri.format)
} else {
default:
fmt.Fprintf(out, " input: env:%s (%s)\n", ri.connEnv, ri.format)
}
}
@@ -339,15 +444,31 @@ func printResolvedJob(out io.Writer, n, total int, rj *resolvedJob) {
} else if rj.outputConnEnv != "" {
fmt.Fprintf(out, " output: env:%s (%s)\n", rj.outputConnEnv, j.Output.Format)
}
if j.Report != nil {
format := valueOr(rj.reportFormat, "default")
if rj.reportPath != "" {
fmt.Fprintf(out, " report: %s (%s)\n", rj.reportPath, format)
} else {
fmt.Fprintf(out, " report: (log) (%s)\n", format)
}
}
if rj.rulesPath != "" {
fmt.Fprintf(out, " rules: %s\n", rj.rulesPath)
} else if j.Command == jobs.CommandInspect {
fmt.Fprintf(out, " rules: (built-in defaults)\n")
}
if rj.selection != nil {
fmt.Fprintf(out, " select: %s\n", rj.selection.summary())
}
if rj.logPath != "" {
fmt.Fprintf(out, " logfile: %s\n", rj.logPath)
fmt.Fprintf(out, " logfile: %s (rotate >= %d bytes, keep %d)\n", rj.logPath, rj.logPolicy.MaxSizeBytes, rj.logPolicy.Keep)
}
fmt.Fprintln(out)
}
// executeResolvedJob runs a single already-validated job.
func executeResolvedJob(rj *resolvedJob) (err error) {
lg, closeLog, lerr := newJobLogger(rj.logPath, rj.secrets)
lg, closeLog, lerr := newJobLogger(rj.logPath, rj.logPolicy, rj.secrets)
if lerr != nil {
return lerr
}
@@ -364,6 +485,14 @@ func executeResolvedJob(rj *resolvedJob) (err error) {
err = runScriptsListJob(rj, lg)
case jobs.CommandTempl:
err = runTemplJob(rj, lg)
case jobs.CommandSplit:
err = runSplitJob(rj, lg)
case jobs.CommandInspect:
err = runInspectJob(rj, lg)
case jobs.CommandDiff:
err = runDiffJob(rj, lg)
case jobs.CommandScriptsExec:
err = runScriptsExecJob(rj, lg)
default:
err = fmt.Errorf("unsupported command %q", rj.job.Command)
}
@@ -449,6 +578,7 @@ func runMergeJob(rj *resolvedJob, lg *jobLogger) error {
lg.logf("merging: %s", inputLabel(ri))
merge.MergeDatabases(base, db, opts)
}
finalizeCommentedRefs(base, func(w string) { lg.logf("warning: %s", w) })
base.UpdateDate()
return writeJobOutput(rj, base, lg)
}
@@ -505,6 +635,168 @@ func runScriptsListJob(rj *resolvedJob, lg *jobLogger) error {
return nil
}
func runSplitJob(rj *resolvedJob, lg *jobLogger) error {
db, err := readJobInputs(rj, lg)
if err != nil {
return err
}
sel := splitSelection{}
if rj.selection != nil {
sel = *rj.selection
}
filtered, err := filterDatabaseSelection(db, sel)
if err != nil {
return fmt.Errorf("split selection: %w", err)
}
if sel.DatabaseName != "" {
filtered.Name = sel.DatabaseName
}
tables := 0
for _, s := range filtered.Schemas {
tables += len(s.Tables)
}
lg.logf("split: selected %d schema(s), %d table(s)", len(filtered.Schemas), tables)
return writeJobOutput(rj, filtered, lg)
}
func runInspectJob(rj *resolvedJob, lg *jobLogger) error {
db, err := readJobInputs(rj, lg)
if err != nil {
return err
}
config, err := inspector.LoadConfig(rj.rulesPath) // "" -> built-in defaults
if err != nil {
return fmt.Errorf("load rules: %w", err)
}
report, err := inspector.NewInspector(db, config).Inspect()
if err != nil {
return fmt.Errorf("inspection failed: %w", err)
}
var formatted string
switch valueOr(rj.reportFormat, "markdown") {
case "json":
formatted, err = inspector.NewJSONFormatter().Format(report)
default:
formatted, err = inspector.NewMarkdownFormatter(io.Discard).Format(report)
}
if err != nil {
return fmt.Errorf("format report: %w", err)
}
if werr := atomicWrite(rj.reportPath, func(tmp string) error {
return os.WriteFile(tmp, []byte(formatted), 0o644)
}); werr != nil {
return werr
}
lg.logf("inspect: %d error(s), %d warning(s) -> %s",
report.Summary.ErrorCount, report.Summary.WarningCount, rj.reportPath)
if report.HasErrors() {
return fmt.Errorf("inspection found %d error(s)", report.Summary.ErrorCount)
}
return nil
}
func runDiffJob(rj *resolvedJob, lg *jobLogger) error {
if len(rj.inputs) != 2 {
return fmt.Errorf("diff requires exactly 2 inputs, got %d", len(rj.inputs))
}
source, err := readOneJobInput(rj.inputs[0])
if err != nil {
return fmt.Errorf("input[0]: %w", err)
}
lg.logf("diff source: %s", inputLabel(rj.inputs[0]))
target, err := readOneJobInput(rj.inputs[1])
if err != nil {
return fmt.Errorf("input[1]: %w", err)
}
lg.logf("diff target: %s", inputLabel(rj.inputs[1]))
result := diff.CompareDatabases(source, target)
s := diff.ComputeSummary(result)
lg.logf("diff: schemas %d/%d/%d, tables %d/%d/%d, columns %d/%d/%d (missing/extra/modified)",
s.Schemas.Missing, s.Schemas.Extra, s.Schemas.Modified,
s.Tables.Missing, s.Tables.Extra, s.Tables.Modified,
s.Columns.Missing, s.Columns.Extra, s.Columns.Modified)
format := diff.FormatSummary
switch rj.reportFormat {
case "json":
format = diff.FormatJSON
case "html":
format = diff.FormatHTML
}
if rj.reportPath == "" {
var buf bytes.Buffer
if err := diff.FormatDiff(result, format, &buf); err != nil {
return fmt.Errorf("format diff: %w", err)
}
for _, line := range strings.Split(strings.TrimRight(buf.String(), "\n"), "\n") {
lg.logf("%s", line)
}
return nil
}
if werr := atomicWrite(rj.reportPath, func(tmp string) error {
f, err := os.Create(tmp)
if err != nil {
return err
}
defer f.Close()
return diff.FormatDiff(result, format, f)
}); werr != nil {
return werr
}
lg.logf("diff report written: %s", rj.reportPath)
return nil
}
func runScriptsExecJob(rj *resolvedJob, lg *jobLogger) error {
schemaName := valueOr(rj.job.Options.Schema, "public")
combined := &models.Schema{Name: schemaName}
for _, dir := range rj.scriptDirs {
reader := sqldir.NewReader(&readers.ReaderOptions{
FilePath: dir,
Metadata: map[string]any{
"schema_name": schemaName,
"database_name": "database",
},
})
db, err := reader.ReadDatabase()
if err != nil {
return fmt.Errorf("%s: %w", dir, err)
}
if len(db.Schemas) == 0 {
continue
}
combined.Scripts = append(combined.Scripts, db.Schemas[0].Scripts...)
}
if len(combined.Scripts) == 0 {
lg.logf("no scripts found; nothing to execute")
return nil
}
lg.logf("executing %d script(s) against database env:%s", len(combined.Scripts), rj.outputConnEnv)
writer := sqlexec.NewWriter(&writers.WriterOptions{
Metadata: map[string]any{
"connection_string": rj.outputConn,
"ignore_errors": rj.job.Options.ContinueOnError,
},
})
if err := writer.WriteSchema(combined); err != nil {
return fmt.Errorf("script execution failed: %w", err)
}
opts := writer.Options()
total, _ := opts.Metadata["execution_total"].(int)
success, _ := opts.Metadata["execution_success"].(int)
failed, _ := opts.Metadata["execution_failed"].(int)
lg.logf("executed %d script(s): %d succeeded, %d failed", total, success, failed)
if failed > 0 && !rj.job.Options.ContinueOnError {
return fmt.Errorf("%d script(s) failed", failed)
}
return nil
}
// readJobInputs reads every input and additively merges them into one model.
func readJobInputs(rj *resolvedJob, lg *jobLogger) (*models.Database, error) {
var base *models.Database
@@ -523,6 +815,7 @@ func readJobInputs(rj *resolvedJob, lg *jobLogger) (*models.Database, error) {
if base == nil {
return nil, fmt.Errorf("no inputs produced a database")
}
finalizeCommentedRefs(base, func(w string) { lg.logf("warning: %s", w) })
return base, nil
}
@@ -550,8 +843,8 @@ func writeJobOutput(rj *resolvedJob, db *models.Database, lg *jobLogger) error {
return fmt.Errorf("database output is only supported for pgsql (got %q)", rj.job.Output.Format)
}
lg.logf("writing output to database env:%s", rj.outputConnEnv)
writerOpts := newWriterOptions("", o.Package, o.FlattenSchema, "", "", o.ContinueOnError)
writerOpts.Metadata = map[string]interface{}{"connection_string": rj.outputConn}
writerOpts := newWriterOptions("", o.Package, o.FlattenSchema, o.Types, o.ArrayNullable, o.ContinueOnError)
writerOpts.Metadata = map[string]interface{}{"connection_string": rj.outputConn, "full_ddl": o.FullDDL}
return wpgsql.NewWriter(writerOpts).WriteDatabase(db)
}
@@ -559,7 +852,37 @@ func writeJobOutput(rj *resolvedJob, db *models.Database, lg *jobLogger) error {
return fmt.Errorf("failed to create output directory: %w", err)
}
lg.logf("writing output: %s (%s)", rj.outputPath, format)
return writeDatabase(db, format, rj.outputPath, o.Package, o.Schema, o.FlattenSchema, "", "", o.ContinueOnError, "")
write := func(target string) error {
return writeDatabase(db, format, target, o.Package, o.Schema, o.FlattenSchema, o.Types, o.ArrayNullable, o.ContinueOnError, "")
}
// Single-file formats are written to a temp file and renamed into place so
// a failure never leaves a partial or truncated output. Directory-emitting
// formats (gorm/bun/drizzle/typeorm/prisma) write in place.
if jobs.SingleFileOutputFormat(format) {
return atomicWrite(rj.outputPath, write)
}
return write(rj.outputPath)
}
// atomicWrite calls produce with a temp path in the same directory as
// finalPath, then renames it over finalPath. The temp file is removed on any
// error so the destination is only ever replaced by a complete file.
func atomicWrite(finalPath string, produce func(tmpPath string) error) error {
dir := filepath.Dir(finalPath)
if err := os.MkdirAll(dir, 0o755); err != nil {
return fmt.Errorf("failed to create output directory: %w", err)
}
tmp := filepath.Join(dir, fmt.Sprintf(".%s.relspec-tmp-%d", filepath.Base(finalPath), os.Getpid()))
if err := produce(tmp); err != nil {
_ = os.Remove(tmp)
return err
}
if err := os.Rename(tmp, finalPath); err != nil {
_ = os.Remove(tmp)
return fmt.Errorf("failed to finalize %s: %w", finalPath, err)
}
return nil
}
// --- logging + redaction ---------------------------------------------------
@@ -572,7 +895,7 @@ type jobLogger struct {
// newJobLogger returns a logger that mirrors to stderr and, when path is set,
// to a job logfile. Connection strings and known secret values are redacted
// from everything it writes.
func newJobLogger(path string, secrets []string) (*jobLogger, func(err error), error) {
func newJobLogger(path string, policy jobs.LogPolicy, secrets []string) (*jobLogger, func(err error), error) {
lg := &jobLogger{secrets: secrets}
if path == "" {
return lg, func(error) {}, nil
@@ -580,6 +903,7 @@ func newJobLogger(path string, secrets []string) (*jobLogger, func(err error), e
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
return nil, nil, fmt.Errorf("failed to create log directory: %w", err)
}
rotateLogIfNeeded(path, policy)
f, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0o644)
if err != nil {
return nil, nil, fmt.Errorf("failed to open logfile %q: %w", path, err)
@@ -593,6 +917,28 @@ func newJobLogger(path string, secrets []string) (*jobLogger, func(err error), e
}, nil
}
// rotateLogIfNeeded renames path -> path.1 -> path.2 ... up to policy.Keep
// when path has grown to policy.MaxSizeBytes or more. The oldest file beyond
// Keep is deleted. A zero/negative MaxSizeBytes disables rotation.
func rotateLogIfNeeded(path string, policy jobs.LogPolicy) {
if policy.MaxSizeBytes <= 0 {
return
}
info, err := os.Stat(path)
if err != nil || info.Size() < policy.MaxSizeBytes {
return
}
if policy.Keep < 1 {
_ = os.Remove(path)
return
}
_ = os.Remove(fmt.Sprintf("%s.%d", path, policy.Keep))
for i := policy.Keep - 1; i >= 1; i-- {
_ = os.Rename(fmt.Sprintf("%s.%d", path, i), fmt.Sprintf("%s.%d", path, i+1))
}
_ = os.Rename(path, path+".1")
}
func (l *jobLogger) logf(format string, args ...interface{}) {
line := l.redact(fmt.Sprintf(format, args...))
fmt.Fprintf(os.Stderr, " %s\n", line)
+339
View File
@@ -376,6 +376,286 @@ jobs:
}
}
func TestJobRun_SplitJob(t *testing.T) {
dir := jobFixture(t, `version: 1
jobs:
extract:
command: split
inputs:
- path: schema/core.dbml
format: dbml
- path: schema/tenant.dbml
format: dbml
select:
tables: [users]
output:
format: json
path: build/subset.json
overwrite: true
`)
set := mustLoadSet(t, filepath.Join(dir, "relspec.yml"))
if err := executeJobPlan(set, "extract", false, false, &bytes.Buffer{}); err != nil {
t.Fatalf("execute split job: %v", err)
}
out, err := os.ReadFile(filepath.Join(dir, "build", "subset.json"))
if err != nil {
t.Fatalf("read split output: %v", err)
}
s := string(out)
if !strings.Contains(s, "users") {
t.Fatalf("split output missing selected table:\n%s", s)
}
if strings.Contains(s, "posts") {
t.Fatalf("split output should have excluded posts:\n%s", s)
}
}
func TestJobRun_InspectJob(t *testing.T) {
dir := jobFixture(t, `version: 1
jobs:
lint:
command: inspect
inputs:
- path: schema/core.dbml
format: dbml
report:
format: json
path: build/report.json
overwrite: true
logfile: .relspec/lint.log
`)
set := mustLoadSet(t, filepath.Join(dir, "relspec.yml"))
// Default rules only warn, so the job succeeds.
if err := executeJobPlan(set, "lint", false, false, &bytes.Buffer{}); err != nil {
t.Fatalf("execute inspect job: %v", err)
}
if _, err := os.ReadFile(filepath.Join(dir, "build", "report.json")); err != nil {
t.Fatalf("expected report file: %v", err)
}
logData, _ := os.ReadFile(filepath.Join(dir, ".relspec", "lint.log"))
if !strings.Contains(string(logData), "inspect:") {
t.Fatalf("logfile missing inspect summary:\n%s", logData)
}
}
func TestJobRun_InspectJobFailsOnRuleError(t *testing.T) {
dir := jobFixture(t, `version: 1
jobs:
lint:
command: inspect
inputs:
- path: schema/core.dbml
format: dbml
rules: rules.yaml
report:
format: json
path: build/report.json
overwrite: true
logfile: .relspec/lint.log
`)
// A rule set to "error" level for a violation the fixture triggers.
writeFile(t, filepath.Join(dir, "rules.yaml"), `version: "1.0"
rules:
primary_key_naming:
enabled: enforce
function: primary_key_naming
pattern: "^id_"
message: "Primary key columns should start with id_"
`)
set := mustLoadSet(t, filepath.Join(dir, "relspec.yml"))
err := executeJobPlan(set, "lint", false, false, &bytes.Buffer{})
if err == nil || !strings.Contains(err.Error(), "error(s)") {
t.Fatalf("expected inspect job to fail on rule error, got %v", err)
}
logData, _ := os.ReadFile(filepath.Join(dir, ".relspec", "lint.log"))
if !strings.Contains(string(logData), "FAILED") {
t.Fatalf("failed inspect job should log FAILED:\n%s", logData)
}
}
func TestJobRun_DiffJob(t *testing.T) {
dir := jobFixture(t, `version: 1
jobs:
compare:
command: diff
inputs:
- path: schema/core.dbml
format: dbml
- path: schema/tenant.dbml
format: dbml
report:
format: json
path: build/diff.json
overwrite: true
`)
set := mustLoadSet(t, filepath.Join(dir, "relspec.yml"))
if err := executeJobPlan(set, "compare", false, false, &bytes.Buffer{}); err != nil {
t.Fatalf("execute diff job: %v", err)
}
out, err := os.ReadFile(filepath.Join(dir, "build", "diff.json"))
if err != nil {
t.Fatalf("read diff report: %v", err)
}
if len(out) == 0 {
t.Fatal("diff report is empty")
}
}
func TestJobRun_FromJobWiring(t *testing.T) {
dir := jobFixture(t, `version: 1
jobs:
a:
command: convert
inputs:
- path: schema/core.dbml
format: dbml
output:
format: json
path: build/a.json
overwrite: true
b:
command: convert
inputs:
- from_job: a
output:
format: yaml
path: build/b.yaml
overwrite: true
`)
set := mustLoadSet(t, filepath.Join(dir, "relspec.yml"))
if err := executeJobPlan(set, "b", false, false, &bytes.Buffer{}); err != nil {
t.Fatalf("execute from_job chain: %v", err)
}
if _, err := os.Stat(filepath.Join(dir, "build", "a.json")); err != nil {
t.Fatalf("producer output missing: %v", err)
}
out, err := os.ReadFile(filepath.Join(dir, "build", "b.yaml"))
if err != nil {
t.Fatalf("consumer output missing: %v", err)
}
if !strings.Contains(string(out), "users") {
t.Fatalf("consumer did not consume producer output:\n%s", out)
}
}
func TestJobRun_LogRotation(t *testing.T) {
dir := jobFixture(t, `version: 1
jobs:
build:
command: scripts-list
script_dirs: [migrations]
log_max_size: "150B"
log_keep: 2
logfile: .relspec/build.log
`)
writeFile(t, filepath.Join(dir, "migrations", "1_001_a.sql"), "CREATE TABLE a();\n")
logPath := filepath.Join(dir, ".relspec", "build.log")
writeFile(t, logPath, strings.Repeat("x", 300)+"\n")
set := mustLoadSet(t, filepath.Join(dir, "relspec.yml"))
if err := executeJobPlan(set, "build", false, false, &bytes.Buffer{}); err != nil {
t.Fatalf("execute job: %v", err)
}
rotated, err := os.ReadFile(logPath + ".1")
if err != nil {
t.Fatalf("expected rotated logfile build.log.1: %v", err)
}
if !strings.Contains(string(rotated), strings.Repeat("x", 300)) {
t.Fatalf("rotated logfile should hold the old content")
}
fresh, err := os.ReadFile(logPath)
if err != nil {
t.Fatalf("expected fresh logfile: %v", err)
}
if strings.Contains(string(fresh), strings.Repeat("x", 300)) {
t.Fatalf("fresh logfile should not contain the rotated-out content:\n%s", fresh)
}
if !strings.Contains(string(fresh), "OK") {
t.Fatalf("fresh logfile should hold the new run:\n%s", fresh)
}
}
func TestJobRun_AtomicOutputLeavesOriginalOnFailure(t *testing.T) {
dir := jobFixture(t, `version: 1
jobs:
x:
command: convert
inputs:
- path: schema/core.dbml
format: dbml
output:
format: json
path: build/out.json
overwrite: true
`)
// Seed the destination, then make its parent directory read-only so the
// rename step fails. The seeded file must survive intact.
seeded := filepath.Join(dir, "build", "out.json")
writeFile(t, seeded, `{"seeded":true}`)
if err := os.Chmod(filepath.Join(dir, "build"), 0o500); err != nil {
t.Skipf("cannot chmod: %v", err)
}
t.Cleanup(func() { _ = os.Chmod(filepath.Join(dir, "build"), 0o755) })
set := mustLoadSet(t, filepath.Join(dir, "relspec.yml"))
if err := executeJobPlan(set, "x", false, false, &bytes.Buffer{}); err == nil {
t.Skip("write unexpectedly succeeded (running as root?)")
}
if err := os.Chmod(filepath.Join(dir, "build"), 0o755); err != nil {
t.Fatal(err)
}
data, err := os.ReadFile(seeded)
if err != nil {
t.Fatalf("seeded file gone: %v", err)
}
if !strings.Contains(string(data), "seeded") {
t.Fatalf("seeded file was corrupted: %s", data)
}
}
func TestJobRun_ScriptsExecMissingConnEnv(t *testing.T) {
dir := jobFixture(t, `version: 1
jobs:
migrate:
command: scripts-exec
script_dirs: [migrations]
output:
conn_env: RELSPEC_TEST_EXEC_MISSING
logfile: .relspec/migrate.log
`)
writeFile(t, filepath.Join(dir, "migrations", "1_001_a.sql"), "CREATE TABLE a();\n")
os.Unsetenv("RELSPEC_TEST_EXEC_MISSING")
set := mustLoadSet(t, filepath.Join(dir, "relspec.yml"))
err := executeJobPlan(set, "migrate", false, false, &bytes.Buffer{})
if err == nil || !strings.Contains(err.Error(), "conn_env") {
t.Fatalf("expected missing conn_env error, got %v", err)
}
}
func TestJobRun_ScriptsExecDryRun(t *testing.T) {
dir := jobFixture(t, `version: 1
jobs:
migrate:
command: scripts-exec
script_dirs: [migrations]
output:
conn_env: RELSPEC_TEST_EXEC_CONN
`)
writeFile(t, filepath.Join(dir, "migrations", "1_001_a.sql"), "CREATE TABLE a();\n")
t.Setenv("RELSPEC_TEST_EXEC_CONN", "postgres://u:secretpw@h/db")
set := mustLoadSet(t, filepath.Join(dir, "relspec.yml"))
var buf bytes.Buffer
if err := executeJobPlan(set, "migrate", true, false, &buf); err != nil {
t.Fatalf("dry run: %v", err)
}
if strings.Contains(buf.String(), "secretpw") {
t.Fatalf("plan leaked secret:\n%s", buf.String())
}
if !strings.Contains(buf.String(), "env:RELSPEC_TEST_EXEC_CONN") {
t.Fatalf("plan should name the env var:\n%s", buf.String())
}
}
func TestJobRun_TemplDatabaseMode(t *testing.T) {
dir := jobFixture(t, `version: 1
jobs:
@@ -386,6 +666,7 @@ jobs:
format: dbml
template: templates/schema.tmpl
output:
format: text
path: build/schema.txt
overwrite: true
`)
@@ -402,3 +683,61 @@ jobs:
t.Fatalf("templ output missing users table: %s", out)
}
}
func TestJobRun_BunSqlTypesOption(t *testing.T) {
dir := jobFixture(t, `version: 1
jobs:
models:
command: convert
inputs:
- path: schema/core.dbml
format: dbml
output:
format: bun
path: build/models
options:
package: models
types: sqltypes
`)
set := mustLoadSet(t, filepath.Join(dir, "relspec.yml"))
if err := executeJobPlan(set, "models", false, false, &bytes.Buffer{}); err != nil {
t.Fatalf("execute Bun job: %v", err)
}
generated, err := os.ReadFile(filepath.Join(dir, "build", "models"))
if err != nil {
t.Fatalf("read Bun output: %v", err)
}
if !strings.Contains(string(generated), "git.warky.dev/wdevs/relspecgo/pkg/sqltypes") {
t.Fatalf("Bun output did not preserve options.types: %s", generated)
}
}
func TestJobValidation_TemplTextFormat(t *testing.T) {
validDir := jobFixture(t, `version: 1
jobs:
docs:
command: templ
inputs: [{path: schema/core.dbml, format: dbml}]
template: schema.tmpl
output: {format: text, path: build/docs.txt}
`)
valid := mustLoadSet(t, filepath.Join(validDir, "relspec.yml"))
if err := valid.Validate(); err != nil {
t.Fatalf("text output should be valid for templ: %v", err)
}
invalidDir := jobFixture(t, `version: 1
jobs:
docs:
command: templ
inputs: [{path: schema/core.dbml, format: dbml}]
template: schema.tmpl
output: {format: json, path: build/docs.txt}
`)
invalid, err := jobs.Load([]string{filepath.Join(invalidDir, "relspec.yml")})
if err != nil {
t.Fatalf("load invalid manifest: %v", err)
}
if err := invalid.Validate(); err == nil || !strings.Contains(err.Error(), "output.format: text") {
t.Fatalf("expected templ format rejection, got %v", err)
}
}
+35 -2
View File
@@ -5,10 +5,43 @@ import (
"os"
)
// asciiLogo (see version.go) is printed by the `version` command.
func main() {
printVersionHeader(os.Args[1:])
if err := rootCmd.Execute(); err != nil {
args := os.Args[1:]
isSilent := hasSilentFlag(args)
if !isSilent {
printVersionHeader(args)
}
previousStderr := os.Stderr
var nullOutput *os.File
if isSilent {
var err error
nullOutput, err = os.OpenFile(os.DevNull, os.O_WRONLY, 0)
if err != nil {
fmt.Fprintln(previousStderr, err)
os.Exit(1)
}
os.Stderr = nullOutput
}
err := rootCmd.Execute()
if nullOutput != nil {
os.Stderr = previousStderr
nullOutput.Close()
}
if err != nil {
fmt.Fprintln(os.Stderr, err)
os.Exit(1)
}
}
func hasSilentFlag(args []string) bool {
for _, arg := range args {
if arg == "--silent" || arg == "--silent=true" {
return true
}
}
return false
}
+45 -3
View File
@@ -59,7 +59,9 @@ var (
mergeSkipTables string // Comma-separated table names to skip
mergeVerbose bool
mergeReportPath string // Path to write merge report
mergeFullDDL bool // Execute full DDL instead of diffing the live pgsql database
mergeFlattenSchema bool
mergeDryRun bool
)
var mergeCmd = &cobra.Command{
@@ -127,7 +129,9 @@ func init() {
mergeCmd.Flags().BoolVar(&mergeSkipSequences, "skip-sequences", false, "Skip sequences during merge")
mergeCmd.Flags().StringVar(&mergeSkipTables, "skip-tables", "", "Comma-separated list of table names to skip during merge")
mergeCmd.Flags().BoolVar(&mergeVerbose, "verbose", false, "Show verbose output")
mergeCmd.Flags().BoolVar(&mergeFullDDL, "full-ddl", false, "pgsql database output: execute the full DDL instead of only the differences from the live database")
mergeCmd.Flags().StringVar(&mergeReportPath, "merge-report", "", "Path to write merge report (JSON format)")
mergeCmd.Flags().BoolVar(&mergeDryRun, "dry-run", false, "Read and merge in memory, then print the merge plan without writing any output")
mergeCmd.Flags().BoolVar(&mergeFlattenSchema, "flatten-schema", false, "Flatten schema.table names to schema_table (useful for databases like SQLite that do not support schemas)")
}
@@ -178,7 +182,7 @@ func runMerge(cmd *cobra.Command, args []string) error {
}
// Step 1: Read target database
fmt.Fprintf(os.Stderr, "[1/3] Reading target database...\n")
fmt.Fprintf(os.Stderr, "[1/4] Reading target database...\n")
fmt.Fprintf(os.Stderr, " Format: %s\n", mergeTargetType)
if mergeTargetPath != "" {
fmt.Fprintf(os.Stderr, " Path: %s\n", mergeTargetPath)
@@ -195,7 +199,7 @@ func runMerge(cmd *cobra.Command, args []string) error {
printDatabaseStats(targetDB)
// Step 2: Read source database(s)
fmt.Fprintf(os.Stderr, "\n[2/3] Reading source database...\n")
fmt.Fprintf(os.Stderr, "\n[2/4] Reading source database...\n")
fmt.Fprintf(os.Stderr, " Format: %s\n", mergeSourceType)
var sourceDB *models.Database
@@ -229,7 +233,7 @@ func runMerge(cmd *cobra.Command, args []string) error {
printDatabaseStats(sourceDB)
// Step 3: Merge databases
fmt.Fprintf(os.Stderr, "\n[3/3] Merging databases...\n")
fmt.Fprintf(os.Stderr, "\n[3/4] Merging databases...\n")
opts := &merge.MergeOptions{
SkipDomains: mergeSkipDomains,
@@ -248,6 +252,7 @@ func runMerge(cmd *cobra.Command, args []string) error {
}
result := merge.MergeDatabases(targetDB, sourceDB, opts)
finalizeCommentedRefs(targetDB, stderrWarn)
// Update timestamp
targetDB.UpdateDate()
@@ -261,12 +266,38 @@ func runMerge(cmd *cobra.Command, args []string) error {
merge.GetColumnTypeConflictSummary(result, 10))
}
if mergeDryRun {
if !isMergeOutputFormat(mergeOutputType) {
return fmt.Errorf("dry run validation failed: Output: unsupported format '%s'", mergeOutputType)
}
out := outWriter(cmd)
fmt.Fprintf(out, "RelSpec merge plan (dry run - nothing written):\n")
fmt.Fprintf(out, " Target: %s database '%s'\n", mergeTargetType, targetDB.Name)
fmt.Fprintf(out, " Source: %s database '%s'\n", mergeSourceType, sourceDB.Name)
switch {
case mergeOutputPath != "":
fmt.Fprintf(out, " Output: %s -> %s\n", mergeOutputType, mergeOutputPath)
case mergeOutputConn != "":
fmt.Fprintf(out, " Output: %s -> %s\n", mergeOutputType, maskPassword(mergeOutputConn))
default:
fmt.Fprintf(out, " Output: %s\n", mergeOutputType)
}
fmt.Fprintf(out, " Result:\n")
printDryRunPlan(out, targetDB)
fmt.Fprintf(out, "\n%s\n", merge.GetMergeSummary(result))
fmt.Fprintf(os.Stderr, "\n=== Dry run complete: no output written ===\n")
return nil
}
// Step 4: Write output
fmt.Fprintf(os.Stderr, "\n[4/4] Writing output...\n")
fmt.Fprintf(os.Stderr, " Format: %s\n", mergeOutputType)
if mergeOutputPath != "" {
fmt.Fprintf(os.Stderr, " Path: %s\n", mergeOutputPath)
}
if mergeOutputConn != "" {
fmt.Fprintf(os.Stderr, " Conn: %s\n", maskPassword(mergeOutputConn))
}
err = writeDatabaseForMerge(mergeOutputType, mergeOutputPath, mergeOutputConn, targetDB, "Output", mergeFlattenSchema)
if err != nil {
@@ -279,6 +310,16 @@ func runMerge(cmd *cobra.Command, args []string) error {
return nil
}
// isMergeOutputFormat reports whether writeDatabaseForMerge supports dbType.
func isMergeOutputFormat(dbType string) bool {
switch strings.ToLower(dbType) {
case "dbml", "dctx", "drawdb", "graphql", "json", "yaml", "gorm", "bun",
"drizzle", "prisma", "typeorm", "sqlite", "sqlite3", "pgsql":
return true
}
return false
}
func readDatabaseForMerge(dbType, filePath, connString, label string) (*models.Database, error) {
var reader readers.Reader
@@ -439,6 +480,7 @@ func writeDatabaseForMerge(dbType, filePath, connString string, db *models.Datab
if connString != "" {
writerOpts.Metadata = map[string]interface{}{
"connection_string": connString,
"full_ddl": mergeFullDDL,
}
// Add report path if merge report is enabled
if mergeReportPath != "" {
+6
View File
@@ -1,6 +1,9 @@
package main
import (
"fmt"
"os"
"git.warky.dev/wdevs/relspecgo/pkg/readers"
"git.warky.dev/wdevs/relspecgo/pkg/writers"
)
@@ -11,6 +14,9 @@ func newReaderOptions(filePath, connString string) *readers.ReaderOptions {
ConnectionString: connString,
Prisma7: prisma7,
StrictDirectives: strictDirectives,
Progress: func(message string) {
fmt.Fprintf(os.Stderr, " → %s\n", message)
},
}
}
+8 -35
View File
@@ -2,51 +2,23 @@ package main
import (
"fmt"
"runtime/debug"
"time"
"github.com/spf13/cobra"
"git.warky.dev/wdevs/relspecgo/pkg/buildinfo"
)
// version/buildDate mirror pkg/buildinfo so existing call sites keep working.
// The actual values are set there via ldflags (see Makefile).
var (
// Version information, set via ldflags during build
version = "dev"
buildDate = "unknown"
version = buildinfo.Version
buildDate = buildinfo.BuildDate
prisma7 bool
noVersion bool
silent bool
strictDirectives bool
)
func init() {
// If version wasn't set via ldflags, try to get it from build info
if version == "dev" {
if info, ok := debug.ReadBuildInfo(); ok {
// Try to get version from VCS
var vcsRevision, vcsTime string
for _, setting := range info.Settings {
switch setting.Key {
case "vcs.revision":
if len(setting.Value) >= 7 {
vcsRevision = setting.Value[:7]
}
case "vcs.time":
vcsTime = setting.Value
}
}
if vcsRevision != "" {
version = vcsRevision
}
if vcsTime != "" {
if t, err := time.Parse(time.RFC3339, vcsTime); err == nil {
buildDate = t.UTC().Format("2006-01-02 15:04:05 UTC")
}
}
}
}
}
var rootCmd = &cobra.Command{
Use: "relspec",
Short: "RelSpec - Database schema conversion and analysis tool",
@@ -73,6 +45,7 @@ func init() {
rootCmd.AddCommand(reportCmd)
rootCmd.PersistentFlags().BoolVar(&prisma7, "prisma7", false, "Use Prisma 7 generator conventions when reading/writing Prisma schemas")
rootCmd.PersistentFlags().BoolVar(&noVersion, "no-version", false, "Suppress the RelSpec version header")
rootCmd.PersistentFlags().BoolVar(&silent, "silent", false, "Suppress progress and status messages (errors are still shown)")
rootCmd.PersistentFlags().BoolVar(&strictDirectives, "strict-directives", false, "Fail on unknown or untranslatable DBML dialect directives (@postgres:, @sqlite:, …)")
}
+73 -6
View File
@@ -24,6 +24,7 @@ var (
splitExcludeTables string
splitNullableTypes string
splitNullableArrays string
splitDryRun bool
)
var splitCmd = &cobra.Command{
@@ -115,6 +116,8 @@ func init() {
splitCmd.Flags().StringVar(&splitNullableTypes, "types", "", "Nullable type package for code-gen writers (bun/gorm): 'baselib' (default, Go pointer types), 'stdlib' (database/sql), or 'sqltypes'")
splitCmd.Flags().StringVar(&splitNullableArrays, "array-nullable", "", "Nullable PostgreSQL array representation for the Bun writer in stdlib/baselib --types mode: 'slice' (default, plain slice) or 'pointer_slice' (*[]T, distinguishes NULL from '{}')")
splitCmd.Flags().BoolVar(&splitDryRun, "dry-run", false, "Read, filter and validate the selection and print the plan without writing any output")
err := splitCmd.MarkFlagRequired("from")
if err != nil {
fmt.Fprintf(os.Stderr, "Error marking from flag as required: %v\n", err)
@@ -174,6 +177,26 @@ func runSplit(cmd *cobra.Command, args []string) error {
}
fmt.Fprintf(os.Stderr, " ✓ Filtered to: %d schema(s), %d table(s)\n\n", len(filteredDB.Schemas), filteredTables)
if splitDryRun {
if err := validateWriteTarget(filteredDB, splitTargetType, splitPackageName, "", ""); err != nil {
return fmt.Errorf("dry run validation failed: %w", err)
}
out := outWriter(cmd)
fmt.Fprintf(out, "RelSpec split plan (dry run - nothing written):\n")
fmt.Fprintf(out, " Input: %s database '%s'\n", splitSourceType, db.Name)
fmt.Fprintf(out, " Output: %s -> %s\n", splitTargetType, splitTargetPath)
fmt.Fprintf(out, " Selection: %s\n", splitSelection{
Schemas: parseCommaSeparated(splitSchemas),
Tables: parseCommaSeparated(splitTables),
ExcludeSchemas: parseCommaSeparated(splitExcludeSchema),
ExcludeTables: parseCommaSeparated(splitExcludeTables),
DatabaseName: splitDatabaseName,
}.summary())
printDryRunPlan(out, filteredDB)
fmt.Fprintf(os.Stderr, "=== Dry run complete: no output written ===\n\n")
return nil
}
// Write to target format
fmt.Fprintf(os.Stderr, "[3/3] Writing to target format...\n")
fmt.Fprintf(os.Stderr, " Format: %s\n", splitTargetType)
@@ -205,8 +228,52 @@ func runSplit(cmd *cobra.Command, args []string) error {
return nil
}
// filterDatabase filters the database based on provided criteria
// splitSelection is the schema/table selection for a split, independent of the
// CLI flag globals so the job runner can build one directly.
type splitSelection struct {
Schemas []string
Tables []string
ExcludeSchemas []string
ExcludeTables []string
DatabaseName string
}
// summary renders a one-line human description of the selection.
func (s splitSelection) summary() string {
var parts []string
if len(s.Schemas) > 0 {
parts = append(parts, "schemas="+strings.Join(s.Schemas, ","))
}
if len(s.Tables) > 0 {
parts = append(parts, "tables="+strings.Join(s.Tables, ","))
}
if len(s.ExcludeSchemas) > 0 {
parts = append(parts, "exclude_schemas="+strings.Join(s.ExcludeSchemas, ","))
}
if len(s.ExcludeTables) > 0 {
parts = append(parts, "exclude_tables="+strings.Join(s.ExcludeTables, ","))
}
if s.DatabaseName != "" {
parts = append(parts, "database_name="+s.DatabaseName)
}
if len(parts) == 0 {
return "(all schemas/tables)"
}
return strings.Join(parts, " ")
}
// filterDatabase filters the database based on the CLI split flags.
func filterDatabase(db *models.Database) (*models.Database, error) {
return filterDatabaseSelection(db, splitSelection{
Schemas: parseCommaSeparated(splitSchemas),
Tables: parseCommaSeparated(splitTables),
ExcludeSchemas: parseCommaSeparated(splitExcludeSchema),
ExcludeTables: parseCommaSeparated(splitExcludeTables),
})
}
// filterDatabaseSelection filters db down to the schemas/tables named by sel.
func filterDatabaseSelection(db *models.Database, sel splitSelection) (*models.Database, error) {
filteredDB := &models.Database{
Name: db.Name,
Description: db.Description,
@@ -220,11 +287,11 @@ func filterDatabase(db *models.Database) (*models.Database, error) {
Domains: db.Domains, // Keep domains for now
}
// Parse filter flags
includeSchemas := parseCommaSeparated(splitSchemas)
includeTables := parseCommaSeparated(splitTables)
excludeSchemas := parseCommaSeparated(splitExcludeSchema)
excludeTables := parseCommaSeparated(splitExcludeTables)
// Selection criteria
includeSchemas := sel.Schemas
includeTables := sel.Tables
excludeSchemas := sel.ExcludeSchemas
excludeTables := sel.ExcludeTables
// Convert table names to lowercase for case-insensitive matching
includeTablesLower := make(map[string]bool)
+1
View File
@@ -114,6 +114,7 @@ func runTempl(cmd *cobra.Command, args []string) error {
if err != nil {
return fmt.Errorf("failed to read source: %w", err)
}
finalizeCommentedRefs(db, stderrWarn)
// Print database stats
schemaCount := len(db.Schemas)
+5 -2
View File
@@ -4,13 +4,16 @@ import (
"fmt"
"github.com/spf13/cobra"
"git.warky.dev/wdevs/relspecgo/pkg/buildinfo"
)
var versionCmd = &cobra.Command{
Use: "version",
Short: "Print version information",
Run: func(cmd *cobra.Command, args []string) {
fmt.Printf("RelSpec %s\n", version)
fmt.Printf("Built: %s\n", buildDate)
fmt.Print(buildinfo.AsciiLogo)
fmt.Printf("RelSpec %s\n", buildinfo.Version)
fmt.Printf("Built: %s\n", buildinfo.BuildDate)
},
}
+103
View File
@@ -0,0 +1,103 @@
# Format Usage Examples
Examples for `relspec convert` covering the file-based reader and writer
formats. The "Writers" and "Readers" sections below were run against
`examples/test_schema.dbml`. The cross-format and live-database examples were
not run; they follow the flags shown in `relspec convert --help` and require
matching input files or reachable databases.
Any reader can be combined with any writer: pick `--from`/`--from-path` for the
source and `--to`/`--to-path` for the target. Add `--silent` to suppress progress
output.
## Writers: DBML to every format
```bash
S="--from dbml --from-path examples/test_schema.dbml"
relspec convert $S --to json --to-path schema.json
relspec convert $S --to yaml --to-path schema.yaml
relspec convert $S --to dctx --to-path schema.dctx
relspec convert $S --to drawdb --to-path schema.drawdb.json
relspec convert $S --to graphql --to-path schema.graphql
relspec convert $S --to prisma --to-path schema.prisma
relspec convert $S --to pgsql --to-path schema.pg.sql
relspec convert $S --to mssql --to-path schema.mssql.sql
relspec convert $S --to sqlite --to-path schema.sqlite.sql
relspec convert $S --to drizzle --to-path schema.ts
relspec convert $S --to typeorm --to-path entities.ts
relspec convert $S --to gorm --to-path models.go --package models
relspec convert $S --to bun --to-path models.go --package models
```
Notes:
- Code-generation writers (`gorm`, `bun`) take `--package`. They also accept
`--types baselib|stdlib|sqltypes` to choose the nullable type package.
- When `--to-path` is a directory it must already exist.
- `sqlite` output automatically flattens `schema.table` names. Use
`--flatten-schema` for other formats if the target has no schema support.
- `dctx` supports a single schema only; use `--schema <name>` to select one.
## Readers: file-based formats into DBML (or JSON where noted)
```bash
relspec convert --from json --from-path schema.json --to dbml --to-path out.dbml
relspec convert --from yaml --from-path schema.yaml --to dbml --to-path out.dbml
relspec convert --from dctx --from-path schema.dctx --to dbml --to-path out.dbml
relspec convert --from drawdb --from-path schema.drawdb.json --to dbml --to-path out.dbml
relspec convert --from graphql --from-path schema.graphql --to dbml --to-path out.dbml
relspec convert --from prisma --from-path schema.prisma --to dbml --to-path out.dbml
relspec convert --from drizzle --from-path schema.ts --to dbml --to-path out.dbml
relspec convert --from typeorm --from-path entities.ts --to dbml --to-path out.dbml
relspec convert --from bun --from-path models.go --to dbml --to-path out.dbml
relspec convert --from gorm --from-path models.go --to json --to-path out.json
```
Code-first readers (`gorm`, `bun`, `drizzle`, `typeorm`) accept a single file or a
directory of model files.
> Known issue: reading GORM models and writing DBML currently panics in the DBML
> writer (`pkg/writers/dbml/writer.go`, `constraintToDBML`). Use another target
> such as JSON until this is fixed.
## Cross-format combinations
```bash
# ORM models to SQL DDL
relspec convert --from gorm --from-path models.go --to pgsql --to-path schema.sql
# Prisma to Drizzle
relspec convert --from prisma --from-path schema.prisma --to drizzle --to-path schema.ts
# DrawDB diagram to GraphQL
relspec convert --from drawdb --from-path diagram.json --to graphql --to-path schema.graphql
# Merge several files while converting
relspec convert --from json --from-list "a.json,b.json" --to yaml --to-path merged.yaml
```
## Live databases
These need a reachable database:
```bash
# PostgreSQL
relspec convert --from pgsql --from-conn "postgres://user:pass@localhost:5432/mydb" \
--to dbml --to-path schema.dbml
# SQL Server
relspec convert --from mssql --from-conn "<mssql connection string>" \
--to json --to-path schema.json
# SQLite database file (--from-conn takes the file path)
relspec convert --from sqlite --from-conn ./app.db --to dbml --to-path schema.dbml
```
## Formats outside `convert`
- `sqldir` (SQL script directory reader) and `sqlexec` (SQL execution writer) are
used by `relspec scripts` and `relspec job`, and `sqldir` by `relspec diff`.
See [SCRIPTS_COMMAND.md](SCRIPTS_COMMAND.md) and [JOB_FILES.md](JOB_FILES.md).
- The `template` writer is exposed through `relspec templ`. See
[TEMPLATE_MODE.md](TEMPLATE_MODE.md).
+174 -30
View File
@@ -9,10 +9,11 @@ relspec job run build-schema --plan # validate + print plan, execute nothing
relspec job run build-schema # run the job (and its dependencies)
```
## Design contract (first release)
## Design contract
This is the smallest coherent contract that is safe and useful end to end.
Anything not listed under "Supported" is intentionally deferred.
This is a deliberately small, safe contract. Every capability is offline-testable
except live database execution (`scripts-exec`), which is validated and planned
offline and only connects at run time.
### Not a shell
@@ -24,14 +25,15 @@ means adding a vetted adapter in the RelSpec source.
|----------------|--------------------------------------------------------------------|
| `convert` | read one or more input schemas, additively merge them, write one output |
| `merge` | like `convert` but requires ≥2 inputs and exposes `skip_*` merge options |
| `split` | read one or more schemas, keep the selected schemas/tables, write one output |
| `scripts-list` | deterministically list SQL scripts across one or more directories |
| `scripts-exec` | execute SQL scripts across one or more directories against a live PostgreSQL database |
| `templ` | apply a custom Go text template to one or more input schemas |
| `inspect` | validate one or more schemas against rules and write a report |
| `diff` | compare exactly two schemas and write a differences report |
Deferred (documented, not implemented here): `scripts` execution against a live
database, `split`, `inspect`, `diff`, job-to-job output wiring,
log rotation/retention. Live SQL execution already exists as
`relspec scripts execute`; wiring it into the job runner is a follow-up because
it needs live database credentials and cannot be covered by offline tests.
`convert`, `merge` and `split` are **producers**: their file output can be fed
directly into another job with `from_job` (see below).
### Discovery and precedence
@@ -49,12 +51,20 @@ already forbid duplicate keys within a single file.
### Paths
* Every path (`inputs[].path`, `output.path`, `script_dirs[]`, `logfile`) is
**relative to the directory containing the job file that declared the job**,
not the process working directory.
* Every path (`inputs[].path`, `output.path`, `report.path`, `rules`,
`script_dirs[]`, `template`, `logfile`) is **relative to the directory
containing the job file that declared the job**, not the process working
directory.
* Those path fields support `${NAME}` environment-variable references. They
are expanded immediately before execution; a missing variable is an error.
The expanded value is still subject to all relative-path and symlink safety
checks.
* Absolute paths, `~`-relative paths and any path that resolves outside the job
file directory (`../`, `a/../../b`, …) are **rejected during validation** —
before anything runs.
* At run time each path is additionally resolved through its symlinks: a symlink
inside the job-file directory that points outside it is rejected before the
path is opened.
### Credentials
@@ -76,79 +86,139 @@ already forbid duplicate keys within a single file.
first. Nothing is read, written, connected to, or executed if validation fails.
Checks include:
* schema `version` (must be `1`), unknown YAML fields rejected
* schema `version` — **forward-permissive**: any version `>= 1` is accepted.
An omitted `version` is treated as the current one. A version newer than this
build understands loads best-effort (unknown YAML fields are ignored and a
warning is printed); at the current version unknown YAML fields are still
rejected.
* duplicate job names across files
* unknown / missing `command`
* per-command input/output shape (`convert`/`merge` need inputs + output;
`scripts-list` needs `script_dirs` and forbids inputs/output)
* per-command input/output shape:
* `convert` needs ≥1 input + output; `merge` needs ≥2 inputs + output
* `split` needs ≥1 input + a file output, plus an optional `select:` block
* `scripts-list` needs `script_dirs` and forbids inputs/output
* `scripts-exec` needs `script_dirs` and `output.conn_env` (pgsql only)
* `inspect` needs ≥1 input + `report:` (format `markdown`|`json`)
* `diff` needs **exactly 2** inputs + `report:` (format `summary`|`json`|`html`)
* unknown input/output `format`
* `from_job` targets exist, are producers (`convert`/`merge`/`split`) and write a
single-file output
* path traversal / absolute / home-relative paths
* `depends_on` targets exist
* dependency cycles (reported as `a -> b -> c -> a`)
* `depends_on` and `from_job` targets exist
* dependency cycles over the combined `depends_on` + `from_job` graph
(reported as `a -> b -> c -> a`)
Then, immediately before running, per-job pre-flight resolves paths and checks:
* every input file exists and is a file
* every input file exists and is a file (a `from_job` input is exempt — its
producer runs earlier in the same plan)
* every `script_dir` exists and is a directory
* every `conn_env` variable is set
* `output.path` does not already exist unless `output.overwrite: true`
* `output.path` / `report.path` does not already exist unless the matching
`overwrite: true` is set
* `rules` (inspect), when given, exists and is a file
* symlinks in every resolved path stay inside the job-file directory
If any pre-flight check fails for **any** job in the plan, **no** job runs.
### Execution and exit codes
* `relspec job run <name>` runs the job's `depends_on` closure first, in
topological order (deterministic), then the job. `--no-deps` runs only the
named job.
* `relspec job run <name>` runs the job's dependency closure first
(`depends_on` plus any `from_job` producers), in topological order
(deterministic), then the job. `--no-deps` runs only the named job and is
incompatible with `from_job` inputs.
* `--dry-run` (alias `--plan`) prints the resolved plan and exits 0 without
touching inputs, outputs or databases.
* A failing job returns the underlying non-zero status (the process exits 1)
and the error names the job. The logfile records `FAILED: <error>`; a
successful job records `OK`. No separate success-marker file is written, so a
failure can never leave a stale "success".
* `inspect` fails the job when the report contains rule **errors** (enforced
rules); warnings do not fail it. `diff` never fails on differences.
* Single-file outputs and reports are written to a temporary file in the target
directory and atomically renamed into place, so an interrupted run never
leaves a partial file. Directory-emitting formats (`gorm`, `bun`, `drizzle`,
`typeorm`, `prisma`) are written in place.
### Logfile rotation
When a job has a `logfile`, it is size-rotated before each run. Defaults are
**5 MB** with **3** rotated files kept (`build.log` → `build.log.1` → …). Override
per job with `log_max_size` / `log_keep`, or for a whole file with a top-level
`defaults:` block. `log_max_size` accepts `B`/`KB`/`MB`/`GB` suffixes (e.g.
`"512KB"`, `"5MB"`).
## Schema reference
```yaml
version: 1 # required, must be 1
version: 1 # optional; any value >= 1 is accepted
defaults: # optional, file-wide
log_max_size: 5MB # B / KB / MB / GB
log_keep: 3
jobs:
<job-name>:
command: convert | merge | scripts-list # required
command: convert | merge | split | scripts-list | scripts-exec | templ | inspect | diff
description: "free text" # optional, shown by `job list`
depends_on: [other-job, ...] # optional
inputs: # convert (≥1) / merge (≥2)
inputs: # convert (≥1) / merge (≥2) / split (≥1) / inspect (≥1) / diff (exactly 2)
- path: relative/file.dbml # file inputs
format: dbml
- format: pgsql # live-connection inputs
conn_env: SOURCE_DB_URL # env var NAME
script_dirs: # scripts-list (≥1)
- from_job: build-schema # consume another job's file output
script_dirs: # scripts-list / scripts-exec (≥1)
- migrations/core
- migrations/tenant
template: templates/schema.tmpl # templ (required)
mode: table # templ: database/schema/script/table
filename_pattern: "{{.Name}}.go" # templ multi-output modes
output: # convert / merge (required)
select: # split (optional; default = keep everything)
schemas: [public]
tables: [users, orders]
exclude_schemas: []
exclude_tables: []
database_name: SubsetDB # optional rename of the output database
rules: .relspec-rules.yaml # inspect (optional; built-in defaults if omitted)
report: # inspect (required) / diff (required)
format: json # inspect: markdown|json ; diff: summary|json|html
path: build/report.json # required, except a diff "summary" (goes to the log)
overwrite: false
output: # convert / merge / split (required); scripts-exec (required, conn_env)
format: pgsql
path: build/schema.sql # file output, OR:
conn_env: TARGET_DB_URL # execute against DB (pgsql only)
# default: reads live DB, executes only the diff
# (file output = full DDL); set options.full_ddl to force full DDL
overwrite: false # default false
options:
flatten_schema: false
schema: public
package: models # for gorm/bun output
continue_on_error: false # pgsql output
types: sqltypes # Bun/GORM nullable types: baselib|stdlib|sqltypes
array_nullable: pointer_slice # Bun: slice|pointer_slice
continue_on_error: false # pgsql / scripts-exec output
full_ddl: false # pgsql database output: true = run full DDL; default diffs against live DB
skip_relations: false # merge only
skip_enums: false
skip_views: false
skip_domains: false
skip_sequences: false
logfile: .relspec/log/<job-name>.log # optional; appended to
logfile: .relspec/log/<job-name>.log # optional; appended to, size-rotated
log_max_size: 5MB # optional per-job override
log_keep: 3 # optional per-job override
```
For `templ`, `inputs` use the same file or `pgsql`/`conn_env` source forms as
schema conversion. `output` is optional (empty means stdout); when present it
contains only `path` and `overwrite`, because templates do not select a schema
writer format.
contains `path` and `overwrite`, plus the explicit `format: text` marker. The
marker is templ-only: it describes arbitrary text produced by a Go template,
not a database schema format. It is optional for compatibility with older
manifests and has no effect on template execution.
A `from_job` input takes no `path`, `format` or `conn_env`: it resolves to the
named job's `output.path` and inherits its format, and implies a dependency on
that job. The producer must be a `convert`, `merge` or `split` job writing a
single-file output.
### Supported input formats
@@ -230,3 +300,77 @@ jobs:
path: snapshots/prod.dbml
overwrite: true
```
### Chain jobs with `from_job`, then lint the result
```yaml
version: 1
jobs:
build-json:
command: convert
inputs:
- { path: schema/core.dbml, format: dbml }
- { path: schema/tenant.dbml, format: dbml }
output: { format: json, path: build/schema.json, overwrite: true }
lint-schema:
command: inspect
inputs:
- from_job: build-json # implies depends_on: [build-json]
rules: .relspec-rules.yaml # optional; built-in rules if omitted
report:
format: markdown
path: build/lint-report.md
overwrite: true
```
`relspec job run lint-schema` runs `build-json` first, then inspects its output.
The job fails (exit 1) if any enforced rule is violated.
### Split a subset out of a larger schema
```yaml
version: 1
jobs:
posts-only:
command: split
inputs:
- { path: schema/core.dbml, format: dbml }
- { path: schema/tenant.dbml, format: dbml }
select:
tables: [posts]
output: { format: dbml, path: build/posts.dbml, overwrite: true }
```
### Diff two schemas
```yaml
version: 1
jobs:
drift:
command: diff
inputs: # exactly two
- { path: build/schema.json, format: json }
- format: pgsql
conn_env: PROD_DB_URL
report:
format: summary # summary → logfile; json/html need a path
```
`diff` reports differences and always exits 0.
### Execute migration scripts against a live database
```yaml
version: 1
jobs:
apply-migrations:
command: scripts-exec
script_dirs:
- migrations/core
- migrations/tenant
output:
conn_env: TARGET_DB_URL # pgsql only; no path
options:
continue_on_error: false
logfile: .relspec/log/apply-migrations.log
```
+80
View File
@@ -0,0 +1,80 @@
# RelSpec job examples
Run these commands from this directory:
```bash
cd examples/jobs
# List every job from relspec.yml and the additional relspec.*.yml files.
relspec job list
# Validate a job and print its resolved plan without reading or writing data.
relspec job run build-schema --plan
# Build PostgreSQL DDL from the two DBML inputs.
relspec job run build-schema
# Run a job's dependency chain. This runs build-schema first, then build-json.
relspec job run build-json
# Inspect build-json's output. The from_job input adds the dependency automatically.
relspec job run lint-schema
# Extract only the posts table.
relspec job run posts-only
# List migration scripts in deterministic order without connecting to a database.
relspec job run migration-order
# Render one TypeScript file per table using the example template.
relspec job run generate-typescript
# Compare the two example schemas and write the summary to the logfile.
relspec job run compare-schemas
# Use an environment variable in an output path.
export VST=build/typescript
relspec job run generate-typescript --plan
relspec job run generate-typescript
```
Database jobs use environment variables for connection strings. Set them in
the shell before running the jobs; do not put connection strings in YAML:
```bash
export SOURCE_DB_URL='postgres://user:password@localhost/source_db'
export TARGET_DB_URL='postgres://user:password@localhost/target_db'
# Read the source database and write a local DBML snapshot.
relspec job run snapshot-database
# Preview migration execution without connecting to PostgreSQL.
relspec job run apply-migrations --plan
# Execute the migrations against TARGET_DB_URL.
relspec job run apply-migrations
```
The `apply-migrations` job demonstrates execution from multiple directories:
```yaml
script_dirs:
- migrations/core
- migrations/tenant
output:
format: pgsql
conn_env: TARGET_DB_URL
```
RelSpec combines the SQL files from both folders in deterministic order before
executing them against the configured PostgreSQL database.
To use a different directory or manifest, pass `--dir` or `--file`:
```bash
relspec job list --dir /path/to/project
relspec job run build-schema --file /path/to/project/relspec.yml --plan
```
See [../../docs/JOB_FILES.md](../../docs/JOB_FILES.md) for the complete job
file reference.
+29
View File
@@ -0,0 +1,29 @@
# Database-oriented examples. Set the referenced environment variables before
# running these jobs; connection strings never belong in a job file.
version: 1
jobs:
snapshot-database:
command: convert
description: Read a PostgreSQL database and write a DBML snapshot
inputs:
- format: pgsql
conn_env: SOURCE_DB_URL
output:
format: dbml
path: build/database-snapshot.dbml
overwrite: true
apply-migrations:
command: scripts-exec
description: Execute migrations from core and tenant folders against PostgreSQL
# All SQL files from these directories are discovered in deterministic order.
script_dirs:
- migrations/core
- migrations/tenant
output:
format: pgsql
conn_env: TARGET_DB_URL
options:
continue_on_error: false
logfile: .relspec/log/apply-migrations.log
+32
View File
@@ -0,0 +1,32 @@
# Reporting and templating examples. These jobs are discovered together with
# relspec.yml when running `relspec job list` from this directory.
version: 1
jobs:
generate-typescript:
command: templ
description: Render one TypeScript file per table from the merged schema
inputs:
- path: schema/core.dbml
format: dbml
- path: schema/tenant.dbml
format: dbml
template: templates/schema.tmpl
mode: table
filename_pattern: "{{.Name}}.ts"
output:
format: text
path: "${VST}"
overwrite: true
compare-schemas:
command: diff
description: Compare the two example schemas and print a summary
inputs:
- path: schema/core.dbml
format: dbml
- path: schema/tenant.dbml
format: dbml
report:
format: summary
logfile: .relspec/log/compare-schemas.log
+34
View File
@@ -4,7 +4,14 @@
# relspec job list
# relspec job run build-schema --plan
# relspec job run build-schema
# relspec job run lint-schema # inspect, consuming build-json's output
version: 1
# File-wide defaults. Individual jobs may override log_max_size / log_keep.
defaults:
log_max_size: 2MB
log_keep: 5
jobs:
build-schema:
command: convert
@@ -43,3 +50,30 @@ jobs:
- migrations/core
- migrations/tenant
logfile: .relspec/log/migration-order.log
lint-schema:
command: inspect
description: Validate build-json's output against the built-in rules
# No depends_on needed: the from_job input implies a dependency on build-json.
inputs:
- from_job: build-json
report:
format: markdown
path: build/lint-report.md
overwrite: true
logfile: .relspec/log/lint-schema.log
posts-only:
command: split
description: Extract just the posts table into its own DBML file
inputs:
- path: schema/core.dbml
format: dbml
- path: schema/tenant.dbml
format: dbml
select:
tables: [posts]
output:
format: dbml
path: build/posts.dbml
overwrite: true
+6
View File
@@ -0,0 +1,6 @@
// Code generated by RelSpec. DO NOT EDIT.
export interface {{.Name}} {
{{- range values .Table.Columns}}
{{.Name}}: {{.Type}};
{{- end}}
}
+2
View File
@@ -4,6 +4,7 @@ go 1.25.13
require (
github.com/gdamore/tcell/v2 v2.13.9
github.com/go-sql-driver/mysql v1.9.3
github.com/google/uuid v1.6.0
github.com/jackc/pgx/v5 v5.9.2
github.com/microsoft/go-mssqldb v1.10.0
@@ -18,6 +19,7 @@ require (
)
require (
filippo.io/edwards25519 v1.1.0 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/dustin/go-humanize v1.0.1 // indirect
github.com/gdamore/encoding v1.0.1 // indirect
+4
View File
@@ -1,3 +1,5 @@
filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA=
filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4=
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.21.1 h1:jHb/wfvRikGdxMXYV3QG/SzUOPYN9KEUUuC0Yd0/vC0=
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.21.1/go.mod h1:pzBXCYn05zvYIrwLgtK8Ap8QcjRg+0i76tMQdWN6wOk=
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.13.1 h1:Hk5QBxZQC1jb2Fwj6mpzme37xbCDdNTxU7O9eb5+LB4=
@@ -21,6 +23,8 @@ github.com/gdamore/encoding v1.0.1 h1:YzKZckdBL6jVt2Gc+5p82qhrGiqMdG/eNs6Wy0u3Uh
github.com/gdamore/encoding v1.0.1/go.mod h1:0Z0cMFinngz9kS1QfMjCP8TY7em3bZYeeklsSDPivEo=
github.com/gdamore/tcell/v2 v2.13.9 h1:uI5l3DYPcFvHINKlGft+en23evOKL+dwtD21QR8ejVA=
github.com/gdamore/tcell/v2 v2.13.9/go.mod h1:+Wfe208WDdB7INEtCsNrAN6O2m+wsTPk1RAovjaILlo=
github.com/go-sql-driver/mysql v1.9.3 h1:U/N249h2WzJ3Ukj8SowVFjdtZKfu9vlLZxjPXV1aweo=
github.com/go-sql-driver/mysql v1.9.3/go.mod h1:qn46aNg1333BRMNU69Lq93t8du/dwxI64Gl8i5p1WMU=
github.com/golang-jwt/jwt/v5 v5.3.1 h1:kYf81DTWFe7t+1VvL7eS+jKFVWaUnK9cB1qbwn63YCY=
github.com/golang-jwt/jwt/v5 v5.3.1/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE=
github.com/golang-sql/civil v0.0.0-20220223132316-b832511892a9 h1:au07oEsX2xN0ktxqI+Sida1w446QrXBRJ0nee3SNZlA=
+2 -2
View File
@@ -1,6 +1,6 @@
# Maintainer: Hein (Warky Devs) <hein@warky.dev>
pkgname=relspec
pkgver=1.0.74
pkgver=1.0.85
pkgrel=1
pkgdesc="RelSpec is a comprehensive database relations management tool that reads, transforms, and writes database table specifications across multiple formats and ORMs."
arch=('x86_64' 'aarch64')
@@ -15,7 +15,7 @@ build() {
export CGO_ENABLED=0
go build \
-trimpath \
-ldflags "-X git.warky.dev/wdevs/relspecgo/cmd/relspec.version=$pkgver" \
-ldflags "-X 'git.warky.dev/wdevs/relspecgo/pkg/buildinfo.Version=v$pkgver' -X 'git.warky.dev/wdevs/relspecgo/pkg/buildinfo.BuildDate=$(date -u +"%Y-%m-%d %H:%M:%S UTC")'" \
-o "$pkgname" ./cmd/relspec
}
+2 -2
View File
@@ -1,5 +1,5 @@
Name: relspec
Version: 1.0.74
Version: 1.0.85
Release: 1%{?dist}
Summary: RelSpec is a comprehensive database relations management tool that reads, transforms, and writes database table specifications across multiple formats and ORMs.
@@ -25,7 +25,7 @@ DBML, GraphQL, and more.
export CGO_ENABLED=0
go build \
-trimpath \
-ldflags "-X git.warky.dev/wdevs/relspecgo/cmd/relspec.version=%{version}" \
-ldflags "-X 'git.warky.dev/wdevs/relspecgo/pkg/buildinfo.Version=v%{version}' -X 'git.warky.dev/wdevs/relspecgo/pkg/buildinfo.BuildDate=$(date -u +"%%Y-%%m-%%d %%H:%%M:%%S UTC")'" \
-o %{name} ./cmd/relspec
%install
+65
View File
@@ -0,0 +1,65 @@
// Package buildinfo exposes the RelSpec version and build date so that both the
// CLI and the schema writers can stamp generated output with the same values.
package buildinfo
import (
"fmt"
"runtime/debug"
"time"
)
// Version and BuildDate are set via -ldflags at build time (see Makefile). When
// built without ldflags they are backfilled from the Go module build info.
var (
Version = "dev"
BuildDate = "unknown"
)
func init() {
if Version != "dev" {
return
}
info, ok := debug.ReadBuildInfo()
if !ok {
return
}
var rev, vcsTime string
for _, s := range info.Settings {
switch s.Key {
case "vcs.revision":
if len(s.Value) >= 7 {
rev = s.Value[:7]
}
case "vcs.time":
vcsTime = s.Value
}
}
if rev != "" {
Version = rev
}
if t, err := time.Parse(time.RFC3339, vcsTime); err == nil {
BuildDate = t.UTC().Format("2006-01-02 15:04:05 UTC")
}
}
// GeneratedComment returns the one-line provenance string embedded in generated
// files, e.g. "RelSpec dev (built: unknown)".
func GeneratedComment() string {
return fmt.Sprintf("RelSpec %s (built: %s)", Version, BuildDate)
}
const AsciiLogo = `
██████╗ ███████╗██╗ ███████╗██████╗ ███████╗ ██████╗
██╔══██╗██╔════╝██║ ██╔════╝██╔══██╗██╔════╝██╔════╝
██████╔╝█████╗ ██║ ███████╗██████╔╝█████╗ ██║
██╔══██╗██╔══╝ ██║ ╚════██║██╔═══╝ ██╔══╝ ██║
██║ ██║███████╗███████╗███████║██║ ███████╗╚██████╗
╚═╝ ╚═╝╚══════╝╚══════╝╚══════╝╚═╝ ╚══════╝ ╚═════╝
[ IN ] ──▶ [ RELSPEC ] ──▶ [ OUT ]
╔══════════════════════════════════════╗
║ ║
║ © WARKY DEVS ║
║ Author: Hein (hein@warky.dev) ║
║ ║
╚══════════════════════════════════════╝
`
+1
View File
@@ -123,6 +123,7 @@ rules:
| `missing_primary_key` | `have_primary_key` | Ensure tables have primary keys |
| `orphaned_foreign_key` | `orphaned_foreign_key` | Detect FKs referencing non-existent tables |
| `circular_dependency` | `circular_dependency` | Detect circular FK dependencies |
| `duplicate_index_name` | `duplicate_index_name` | Index / PK / unique names must be unique per schema (default: `enforce`) |
## Rule Configuration
+1
View File
@@ -161,6 +161,7 @@ func getValidator(functionName string) (validatorFunc, bool) {
"have_primary_key": validateMissingPrimaryKey,
"orphaned_foreign_key": validateOrphanedForeignKey,
"circular_dependency": validateCircularDependency,
"duplicate_index_name": validateDuplicateIndexName,
}
fn, exists := validators[functionName]
+5
View File
@@ -154,6 +154,11 @@ func GetDefaultConfig() *Config {
Function: "circular_dependency",
Message: "Circular foreign key dependency detected",
},
"duplicate_index_name": {
Enabled: "enforce",
Function: "duplicate_index_name",
Message: "Index name is reused within the schema; PostgreSQL skips the duplicate CREATE INDEX IF NOT EXISTS",
},
},
}
}
+1
View File
@@ -37,6 +37,7 @@ func TestGetDefaultConfig(t *testing.T) {
"missing_primary_key",
"orphaned_foreign_key",
"circular_dependency",
"duplicate_index_name",
}
for _, ruleName := range expectedRules {
+61
View File
@@ -643,3 +643,64 @@ func contains(slice []string, value string) bool {
}
return false
}
// validateDuplicateIndexName checks that index names are unique per schema.
// PostgreSQL keeps indexes in the schema-wide relation namespace, so a name
// reused on another table makes CREATE INDEX IF NOT EXISTS silently skip it.
// Primary key and unique constraints create backing indexes and share that
// namespace too. An index and a constraint with the same name on the same
// table describe one object and are not reported.
func validateDuplicateIndexName(db *models.Database, rule Rule, ruleName string) []ValidationResult {
results := []ValidationResult{}
for _, schema := range db.Schemas {
// lowercased name -> "table" entries, one per distinct object
owners := make(map[string][]string)
display := make(map[string]string)
order := []string{}
add := func(name, table string, sameTableMerges bool) {
key := strings.ToLower(name)
if _, seen := owners[key]; !seen {
order = append(order, key)
display[key] = name
}
if sameTableMerges && contains(owners[key], table) {
return
}
owners[key] = append(owners[key], table)
}
for _, table := range schema.Tables {
for _, key := range sortedKeys(table.Indexes) {
if name := table.Indexes[key].Name; name != "" {
add(name, table.Name, false)
}
}
for _, c := range sortConstraints(table.Constraints) {
if c.Name == "" || (c.Type != models.PrimaryKeyConstraint && c.Type != models.UniqueConstraint) {
continue
}
add(c.Name, table.Name, true)
}
}
for _, key := range order {
tables := owners[key]
results = append(results, createResult(
ruleName,
len(tables) == 1,
rule.Message,
formatLocation(schema.Name, display[key], "")+" on "+strings.Join(tables, ", "),
map[string]interface{}{
"schema": schema.Name,
"index": display[key],
"tables": tables,
"occurrences": len(tables),
},
))
}
}
return results
}
+81
View File
@@ -835,3 +835,84 @@ func TestFormatLocation(t *testing.T) {
})
}
}
func TestValidateDuplicateIndexName(t *testing.T) {
db := &models.Database{
Name: "testdb",
Schemas: []*models.Schema{
{
Name: "entity",
Tables: []*models.Table{
{
Name: "actor_phone",
Indexes: map[string]*models.Index{
"idx_actor": {Name: "idx_actor", Columns: []string{"rid_actor"}},
"uk_phone": {Name: "uk_phone", Columns: []string{"phone"}, Unique: true},
},
Constraints: map[string]*models.Constraint{
// Same name as the index on the same table: one object.
"uk_phone": {Name: "uk_phone", Type: models.UniqueConstraint, Columns: []string{"phone"}},
},
},
{
Name: "actor_email",
Indexes: map[string]*models.Index{
"idx_actor": {Name: "idx_actor", Columns: []string{"rid_actor"}},
},
Constraints: map[string]*models.Constraint{
"UK_Phone": {Name: "UK_Phone", Type: models.UniqueConstraint, Columns: []string{"email"}},
},
},
{
Name: "actor_address",
Indexes: map[string]*models.Index{
"idx_actor": {Name: "idx_actor", Columns: []string{"rid_actor"}},
"idx_actor#2": {Name: "idx_actor", Columns: []string{"rid_actor", "kind"}},
"idx_address": {Name: "idx_address", Columns: []string{"line1"}},
},
},
},
},
{
// Same names in another schema do not collide.
Name: "org",
Tables: []*models.Table{
{Name: "api_provider", Indexes: map[string]*models.Index{"idx_actor": {Name: "idx_actor"}}},
},
},
},
}
results := validateDuplicateIndexName(db, Rule{Message: "dup"}, "duplicate_index_name")
got := map[string]bool{}
occ := map[string]int{}
for _, r := range results {
key := r.Context["schema"].(string) + "." + r.Context["index"].(string)
got[key] = r.Passed
occ[key] = r.Context["occurrences"].(int)
}
want := map[string]struct {
passed bool
occ int
}{
"entity.idx_actor": {false, 4},
"entity.uk_phone": {false, 2},
"entity.idx_address": {true, 1},
"org.idx_actor": {true, 1},
}
if len(got) != len(want) {
t.Fatalf("got %d results %v, want %d", len(got), got, len(want))
}
for k, w := range want {
p, ok := got[k]
if !ok {
t.Errorf("missing result for %s", k)
continue
}
if p != w.passed || occ[k] != w.occ {
t.Errorf("%s: passed=%v occurrences=%d, want passed=%v occurrences=%d", k, p, occ[k], w.passed, w.occ)
}
}
}
+431 -23
View File
@@ -19,25 +19,54 @@ import (
"fmt"
"os"
"path/filepath"
"regexp"
"sort"
"strconv"
"strings"
"gopkg.in/yaml.v3"
)
// SchemaVersion is the only job-file schema version this build understands.
const SchemaVersion = 1
// CurrentSchemaVersion is the highest job-file schema version this build was
// written for. MinSchemaVersion is the oldest it still accepts. A file that
// declares a version in between loads normally; a newer version loads
// best-effort with a warning (see Load); an older-than-minimum version is a
// hard error.
const (
CurrentSchemaVersion = 1
MinSchemaVersion = 1
)
// Built-in logfile rotation policy, used when neither the job nor its file's
// defaults block sets one.
const (
defaultLogMaxSizeBytes int64 = 5 << 20 // 5 MiB
defaultLogKeep = 3
)
// Command names are a closed allow-list. Arbitrary strings are rejected.
const (
CommandConvert = "convert" // read one or more schema files, optionally merge, write one output
CommandMerge = "merge" // additive merge of two or more schema files into one output
CommandScriptsList = "scripts-list" // deterministically list SQL scripts across one or more directories
CommandScriptsExec = "scripts-exec" // execute SQL scripts across one or more directories against a live database
CommandTempl = "templ" // apply a custom Go text template to one or more schemas
CommandSplit = "split" // extract selected schemas/tables into a separate output
CommandInspect = "inspect" // validate one or more schemas against rules and write a report
CommandDiff = "diff" // compare exactly two schemas and write a differences report
)
// SupportedCommands lists every accepted command, in help order.
var SupportedCommands = []string{CommandConvert, CommandMerge, CommandScriptsList, CommandTempl}
var SupportedCommands = []string{
CommandConvert, CommandMerge, CommandScriptsList, CommandScriptsExec,
CommandTempl, CommandSplit, CommandInspect, CommandDiff,
}
// producerCommands are commands whose output is a schema file that another job
// may consume via from_job.
var producerCommands = map[string]bool{
CommandConvert: true, CommandMerge: true, CommandSplit: true,
}
// readerFormats are the file-based input formats a job may declare (path).
var readerFormats = map[string]bool{
@@ -61,10 +90,41 @@ var writerFormats = map[string]bool{
// live database) is supported instead of writing a file.
var execOutputFormats = map[string]bool{"pgsql": true}
// singleFileFormats are output formats that emit exactly one file (as opposed
// to a directory of files). Only these are eligible for atomic temp+rename
// writes and for being consumed by another job via from_job.
var singleFileFormats = map[string]bool{
"json": true, "yaml": true, "dbml": true, "dctx": true, "drawdb": true,
"graphql": true, "pgsql": true, "mssql": true, "sqlite": true,
}
// SingleFileOutputFormat reports whether format writes exactly one file.
func SingleFileOutputFormat(format string) bool {
return singleFileFormats[strings.ToLower(format)]
}
// diffReportFormats and inspectReportFormats are the report.format values
// accepted by the diff and inspect commands respectively.
var (
diffReportFormats = map[string]bool{"summary": true, "json": true, "html": true}
inspectReportFormats = map[string]bool{"markdown": true, "json": true}
)
// File is the on-disk shape of a single job file.
type File struct {
Version int `yaml:"version"`
Jobs map[string]*Job `yaml:"jobs"`
Version int `yaml:"version"`
Defaults *Defaults `yaml:"defaults"`
Jobs map[string]*Job `yaml:"jobs"`
}
// Defaults carries file-wide settings that individual jobs may override.
type Defaults struct {
// LogMaxSize is a human-readable size ("5MB", "512KB", "1GB"). Empty
// means "use the built-in default".
LogMaxSize string `yaml:"log_max_size"`
// LogKeep is how many rotated logfiles to retain. Zero means "use the
// built-in default".
LogKeep int `yaml:"log_keep"`
}
// Job is one named job within a job file.
@@ -72,6 +132,9 @@ type Job struct {
// Name and SourceFile are populated by Load, not parsed from YAML.
Name string `yaml:"-"`
SourceFile string `yaml:"-"`
// fileDefaults is the Defaults block of the file that declared this job,
// captured by Load. nil when the file had none.
fileDefaults *Defaults `yaml:"-"`
Command string `yaml:"command"`
Description string `yaml:"description"`
@@ -82,8 +145,13 @@ type Job struct {
Mode string `yaml:"mode"`
FilenamePattern string `yaml:"filename_pattern"`
Output *Output `yaml:"output"`
Rules string `yaml:"rules"`
Report *Report `yaml:"report"`
Select *Select `yaml:"select"`
Options Options `yaml:"options"`
Logfile string `yaml:"logfile"`
LogMaxSize string `yaml:"log_max_size"`
LogKeep *int `yaml:"log_keep"`
}
// Input is one declared input schema.
@@ -94,6 +162,10 @@ type Input struct {
// ConnEnv is the NAME of an environment variable holding a connection
// string, used with database formats. The value is never stored here.
ConnEnv string `yaml:"conn_env"`
// FromJob names another job in the set whose file output is used as this
// input. It implies a dependency on that job. Path/Format/ConnEnv must be
// empty when FromJob is set; the format is inherited from the producer.
FromJob string `yaml:"from_job"`
}
// Output is the declared output target.
@@ -104,17 +176,123 @@ type Output struct {
Overwrite bool `yaml:"overwrite"`
}
// Report is the output target for the inspect and diff commands.
type Report struct {
// Format is the report format: diff accepts summary|json|html, inspect
// accepts markdown|json. Empty means the command's default.
Format string `yaml:"format"`
Path string `yaml:"path"`
Overwrite bool `yaml:"overwrite"`
}
// Select carries the schema/table selection for the split command.
type Select struct {
Schemas []string `yaml:"schemas"`
Tables []string `yaml:"tables"`
ExcludeSchemas []string `yaml:"exclude_schemas"`
ExcludeTables []string `yaml:"exclude_tables"`
DatabaseName string `yaml:"database_name"`
}
// LogPolicy is the resolved logfile rotation policy for a job.
type LogPolicy struct {
MaxSizeBytes int64
Keep int
}
// ResolvedLogPolicy returns the effective rotation policy: the job's own
// overrides win, then its file's defaults block, then the built-in default.
func (j *Job) ResolvedLogPolicy() LogPolicy {
p := LogPolicy{MaxSizeBytes: defaultLogMaxSizeBytes, Keep: defaultLogKeep}
if j.fileDefaults != nil {
if n, err := parseHumanSize(j.fileDefaults.LogMaxSize); err == nil && n > 0 {
p.MaxSizeBytes = n
}
if j.fileDefaults.LogKeep > 0 {
p.Keep = j.fileDefaults.LogKeep
}
}
if n, err := parseHumanSize(j.LogMaxSize); err == nil && n > 0 {
p.MaxSizeBytes = n
}
if j.LogKeep != nil && *j.LogKeep >= 0 {
p.Keep = *j.LogKeep
}
return p
}
// effectiveDeps returns the union of explicit depends_on entries and the jobs
// referenced by from_job inputs, deduplicated in stable order.
func (j *Job) effectiveDeps() []string {
seen := map[string]bool{}
var deps []string
add := func(name string) {
if name == "" || name == j.Name || seen[name] {
return
}
seen[name] = true
deps = append(deps, name)
}
for _, d := range j.DependsOn {
add(d)
}
for _, in := range j.Inputs {
add(in.FromJob)
}
return deps
}
// parseHumanSize parses a byte size such as "5MB", "512 KB", "1gb" or a bare
// byte count. An empty string returns (0, nil) so callers can fall back.
func parseHumanSize(s string) (int64, error) {
s = strings.TrimSpace(s)
if s == "" {
return 0, nil
}
upper := strings.ToUpper(s)
mult := int64(1)
// Check multi-character suffixes before the bare "B".
for _, u := range []struct {
suffix string
m int64
}{
{"KB", 1 << 10}, {"MB", 1 << 20}, {"GB", 1 << 30}, {"B", 1},
} {
if strings.HasSuffix(upper, u.suffix) {
mult = u.m
upper = strings.TrimSpace(strings.TrimSuffix(upper, u.suffix))
break
}
}
n, err := strconv.ParseFloat(upper, 64)
if err != nil {
return 0, fmt.Errorf("invalid size %q", s)
}
if n < 0 {
return 0, fmt.Errorf("negative size %q", s)
}
return int64(n * float64(mult)), nil
}
// Options carries the subset of command flags a job file may set.
type Options struct {
FlattenSchema bool `yaml:"flatten_schema"`
Schema string `yaml:"schema"`
Package string `yaml:"package"`
FlattenSchema bool `yaml:"flatten_schema"`
Schema string `yaml:"schema"`
Package string `yaml:"package"`
// Types and ArrayNullable mirror the existing convert/split writer flags.
// They are passed through only to writers that already support them.
Types string `yaml:"types"`
ArrayNullable string `yaml:"array_nullable"`
ContinueOnError bool `yaml:"continue_on_error"`
SkipRelations bool `yaml:"skip_relations"`
SkipEnums bool `yaml:"skip_enums"`
SkipViews bool `yaml:"skip_views"`
SkipDomains bool `yaml:"skip_domains"`
SkipSequences bool `yaml:"skip_sequences"`
// FullDDL makes direct pgsql output execute the full idempotent DDL instead of
// diffing against the live database first.
FullDDL bool `yaml:"full_ddl"`
SkipRelations bool `yaml:"skip_relations"`
SkipEnums bool `yaml:"skip_enums"`
SkipViews bool `yaml:"skip_views"`
SkipDomains bool `yaml:"skip_domains"`
SkipSequences bool `yaml:"skip_sequences"`
}
// Dir returns the directory that a job's relative paths resolve against:
@@ -127,6 +305,9 @@ type Set struct {
Files []string
// Jobs is keyed by job name.
Jobs map[string]*Job
// Warnings holds non-fatal load-time messages (e.g. a newer-than-known
// schema version). Callers should surface these to the user.
Warnings []string
}
// Names returns all job names in deterministic (sorted) order.
@@ -199,15 +380,35 @@ func Load(paths []string) (*Set, error) {
if err != nil {
return nil, fmt.Errorf("failed to read job file %q: %w", path, err)
}
// Peek at the version first so a newer file can be parsed leniently
// (unknown fields ignored) instead of failing outright.
var probe struct {
Version int `yaml:"version"`
}
if err := yaml.Unmarshal(data, &probe); err != nil {
return nil, fmt.Errorf("invalid job file %q: %w", path, err)
}
version := probe.Version
if version == 0 {
version = CurrentSchemaVersion
}
if version < MinSchemaVersion {
return nil, fmt.Errorf("job file %q: unsupported version %d (this build accepts %d or newer)", path, version, MinSchemaVersion)
}
strict := version <= CurrentSchemaVersion
if !strict {
set.Warnings = append(set.Warnings, fmt.Sprintf(
"job file %q declares version %d, newer than this build understands (%d); loading best-effort and ignoring unknown fields",
path, version, CurrentSchemaVersion))
}
dec := yaml.NewDecoder(strings.NewReader(string(data)))
dec.KnownFields(true)
dec.KnownFields(strict)
var f File
if err := dec.Decode(&f); err != nil {
return nil, fmt.Errorf("invalid job file %q: %w", path, err)
}
if f.Version != SchemaVersion {
return nil, fmt.Errorf("job file %q: unsupported version %d (expected %d)", path, f.Version, SchemaVersion)
}
if len(f.Jobs) == 0 {
return nil, fmt.Errorf("job file %q: no jobs defined", path)
}
@@ -220,6 +421,7 @@ func Load(paths []string) (*Set, error) {
}
job.Name = name
job.SourceFile = path
job.fileDefaults = f.Defaults
origin[name] = path
set.Jobs[name] = job
}
@@ -240,13 +442,30 @@ func (s *Set) Validate() error {
errs = append(errs, fmt.Sprintf("job %q: %s", name, msg))
}
}
// Dependency references + cycles.
// Dependency references + cycles + from_job wiring.
for _, name := range s.Names() {
for _, dep := range s.Jobs[name].DependsOn {
j := s.Jobs[name]
for _, dep := range j.DependsOn {
if _, ok := s.Jobs[dep]; !ok {
errs = append(errs, fmt.Sprintf("job %q: depends_on unknown job %q", name, dep))
}
}
for i, in := range j.Inputs {
if in.FromJob == "" {
continue
}
producer, ok := s.Jobs[in.FromJob]
if !ok {
errs = append(errs, fmt.Sprintf("job %q: input[%d] from_job references unknown job %q", name, i, in.FromJob))
continue
}
if !producerCommands[producer.Command] || producer.Output == nil ||
producer.Output.Path == "" || !SingleFileOutputFormat(producer.Output.Format) {
errs = append(errs, fmt.Sprintf(
"job %q: input[%d] from_job %q must name a convert/merge/split job that writes a single-file output",
name, i, in.FromJob))
}
}
}
if cycle := s.findCycle(); cycle != "" {
errs = append(errs, fmt.Sprintf("dependency cycle detected: %s", cycle))
@@ -262,7 +481,8 @@ func (j *Job) validate() []string {
var e []string
switch j.Command {
case CommandConvert, CommandMerge, CommandScriptsList, CommandTempl:
case CommandConvert, CommandMerge, CommandScriptsList, CommandScriptsExec,
CommandTempl, CommandSplit, CommandInspect, CommandDiff:
case "":
e = append(e, "missing command")
return e
@@ -282,6 +502,7 @@ func (j *Job) validate() []string {
}
checkPath("logfile", j.Logfile)
checkPath("template", j.Template)
checkPath("rules", j.Rules)
for _, in := range j.Inputs {
checkPath("input path", in.Path)
}
@@ -291,6 +512,13 @@ func (j *Job) validate() []string {
if j.Output != nil {
checkPath("output path", j.Output.Path)
}
if j.Report != nil {
checkPath("report path", j.Report.Path)
}
if _, err := parseHumanSize(j.LogMaxSize); err != nil {
e = append(e, fmt.Sprintf("log_max_size: %v", err))
}
switch j.Command {
case CommandConvert, CommandMerge:
@@ -348,13 +576,117 @@ func (j *Job) validate() []string {
e = append(e, "command \"templ\" does not support database output")
}
if j.Output != nil && j.Output.Format != "" {
e = append(e, "output.format is not valid for command \"templ\"")
if !strings.EqualFold(j.Output.Format, "text") {
e = append(e, "command \"templ\" accepts only output.format: text")
}
}
case CommandSplit:
if len(j.Inputs) < 1 {
e = append(e, "command \"split\" requires at least 1 input")
}
for i, in := range j.Inputs {
e = append(e, validateInput(i, in)...)
}
if len(j.ScriptDirs) > 0 {
e = append(e, "script_dirs is not valid for command \"split\"")
}
if j.Report != nil {
e = append(e, "report is not valid for command \"split\" (use output)")
}
if j.Output == nil {
e = append(e, "missing output")
} else {
if j.Output.ConnEnv != "" {
e = append(e, "command \"split\" writes a file; output.conn_env is not supported")
}
e = append(e, validateOutput(*j.Output)...)
}
case CommandInspect:
if len(j.Inputs) < 1 {
e = append(e, "command \"inspect\" requires at least 1 input")
}
for i, in := range j.Inputs {
e = append(e, validateInput(i, in)...)
}
if len(j.ScriptDirs) > 0 {
e = append(e, "script_dirs is not valid for command \"inspect\"")
}
if j.Output != nil {
e = append(e, "output is not valid for command \"inspect\" (use report)")
}
e = append(e, validateReport(j.Report, "inspect", inspectReportFormats, "markdown")...)
case CommandDiff:
if len(j.Inputs) != 2 {
e = append(e, "command \"diff\" requires exactly 2 inputs (source, target)")
}
for i, in := range j.Inputs {
e = append(e, validateInput(i, in)...)
}
if len(j.ScriptDirs) > 0 {
e = append(e, "script_dirs is not valid for command \"diff\"")
}
if j.Output != nil {
e = append(e, "output is not valid for command \"diff\" (use report)")
}
e = append(e, validateReport(j.Report, "diff", diffReportFormats, "summary")...)
case CommandScriptsExec:
if len(j.ScriptDirs) == 0 {
e = append(e, "command \"scripts-exec\" requires at least one script_dir")
}
if len(j.Inputs) > 0 {
e = append(e, "inputs is not valid for command \"scripts-exec\"")
}
if j.Report != nil {
e = append(e, "report is not valid for command \"scripts-exec\"")
}
if j.Output == nil || j.Output.ConnEnv == "" {
e = append(e, "command \"scripts-exec\" requires output.conn_env (an environment variable name holding a connection string)")
} else {
if j.Output.Path != "" {
e = append(e, "command \"scripts-exec\" executes against a database; output.path is not supported")
}
f := strings.ToLower(j.Output.Format)
if f != "" && f != "pgsql" {
e = append(e, fmt.Sprintf("command \"scripts-exec\" only supports pgsql databases (got %q)", j.Output.Format))
}
if looksLikeSecret(j.Output.ConnEnv) {
e = append(e, "output: conn_env must be an environment variable name, not a connection string")
}
}
}
return e
}
// validateReport checks a Report block for the inspect/diff commands.
func validateReport(r *Report, cmd string, allowed map[string]bool, defFmt string) []string {
if r == nil {
return []string{fmt.Sprintf("command %q requires a report block", cmd)}
}
var e []string
f := strings.ToLower(r.Format)
if f == "" {
f = defFmt
}
if !allowed[f] {
names := make([]string, 0, len(allowed))
for k := range allowed {
names = append(names, k)
}
sort.Strings(names)
e = append(e, fmt.Sprintf("command %q report.format %q is not supported (use: %s)", cmd, r.Format, strings.Join(names, ", ")))
}
// A diff summary may be written to the log; everything else needs a path.
summaryToLog := cmd == "diff" && f == "summary"
if r.Path == "" && !summaryToLog {
e = append(e, fmt.Sprintf("command %q requires report.path", cmd))
}
return e
}
func validateTemplInput(i int, in Input) []string {
if in.FromJob != "" {
return fromJobInputShape(i, in)
}
var e []string
if in.Format == "" {
return []string{fmt.Sprintf("input[%d]: missing format", i)}
@@ -383,7 +715,27 @@ func validateTemplInput(i int, in Input) []string {
return e
}
// fromJobInputShape checks the structural rules for an input that pulls its
// schema from another job's output. The referenced job's existence and kind
// are checked in Set.Validate, which can see the whole set.
func fromJobInputShape(i int, in Input) []string {
var e []string
if in.Path != "" {
e = append(e, fmt.Sprintf("input[%d]: from_job takes no path", i))
}
if in.Format != "" {
e = append(e, fmt.Sprintf("input[%d]: from_job inherits the producer's format; drop format", i))
}
if in.ConnEnv != "" {
e = append(e, fmt.Sprintf("input[%d]: from_job takes no conn_env", i))
}
return e
}
func validateInput(i int, in Input) []string {
if in.FromJob != "" {
return fromJobInputShape(i, in)
}
var e []string
if in.Format == "" {
e = append(e, fmt.Sprintf("input[%d]: missing format", i))
@@ -487,9 +839,65 @@ func SafeJoin(root, rel string) (string, error) {
if rp == ".." || strings.HasPrefix(rp, ".."+string(filepath.Separator)) {
return "", fmt.Errorf("path %q escapes the job file directory", rel)
}
// Symlink hardening: resolve symlinks on the root and on the deepest
// existing ancestor of the target, and require the target to still live
// inside the resolved root. This catches a symlink inside the job-file
// directory that points outside it.
realRoot, err := filepath.EvalSymlinks(absRoot)
if err != nil {
return "", fmt.Errorf("cannot resolve job file directory: %w", err)
}
realAnc, err := filepath.EvalSymlinks(deepestExistingAncestor(joined))
if err != nil {
return "", fmt.Errorf("cannot resolve path %q: %w", rel, err)
}
if realAnc != realRoot {
if r, err := filepath.Rel(realRoot, realAnc); err != nil ||
r == ".." || strings.HasPrefix(r, ".."+string(filepath.Separator)) {
return "", fmt.Errorf("path %q resolves outside the job file directory via a symlink", rel)
}
}
return joined, nil
}
var envReferencePattern = regexp.MustCompile(`\$\{([A-Za-z_][A-Za-z0-9_]*)\}`)
// ExpandEnv expands ${NAME} references using the current process environment.
// It deliberately supports only shell-independent variable references; job
// files are never passed through a shell. An unset variable is an error so a
// typo cannot silently turn into a relative path.
func ExpandEnv(value string) (string, error) {
var missing string
expanded := envReferencePattern.ReplaceAllStringFunc(value, func(reference string) string {
name := reference[2 : len(reference)-1]
resolved, ok := os.LookupEnv(name)
if !ok {
missing = name
return reference
}
return resolved
})
if missing != "" {
return "", fmt.Errorf("environment variable %q referenced by ${%s} is not set", missing, missing)
}
return expanded, nil
}
// deepestExistingAncestor returns p itself if it exists, otherwise the nearest
// existing parent directory (falling back to the filesystem root).
func deepestExistingAncestor(p string) string {
for {
if _, err := os.Lstat(p); err == nil {
return p
}
parent := filepath.Dir(p)
if parent == p {
return p
}
p = parent
}
}
// Plan returns the jobs to execute for name in dependency order. When
// includeDeps is false only the named job is returned (its declared
// dependencies are still validated to exist and be acyclic by Validate).
@@ -514,7 +922,7 @@ func (s *Set) Plan(name string, includeDeps bool) ([]*Job, error) {
}
inProgress[n] = true
j := s.Jobs[n]
deps := append([]string(nil), j.DependsOn...)
deps := j.effectiveDeps()
sort.Strings(deps)
for _, d := range deps {
if _, ok := s.Jobs[d]; !ok {
@@ -543,7 +951,7 @@ func (s *Set) findCycle() string {
dfs = func(n string) []string {
color[n] = 1
stack = append(stack, n)
deps := append([]string(nil), s.Jobs[n].DependsOn...)
deps := s.Jobs[n].effectiveDeps()
sort.Strings(deps)
for _, d := range deps {
if _, ok := s.Jobs[d]; !ok {
+218 -5
View File
@@ -56,13 +56,97 @@ func TestLoadRejectsUnknownFields(t *testing.T) {
}
}
func TestLoadRejectsBadVersion(t *testing.T) {
func TestLoadWarnsOnNewerVersion(t *testing.T) {
dir := t.TempDir()
p := filepath.Join(dir, "relspec.yml")
write(t, p, "version: 2\njobs:\n a:\n command: convert\n")
_, err := Load([]string{p})
if err == nil || !strings.Contains(err.Error(), "unsupported version") {
t.Fatalf("expected unsupported version error, got %v", err)
// A newer version loads best-effort with a warning, and unknown fields
// from the newer schema are ignored rather than rejected.
write(t, p, "version: 99\njobs:\n a:\n command: convert\n"+
" inputs:\n - path: a.dbml\n format: dbml\n"+
" output:\n format: json\n path: out.json\n"+
" future_field: whatever\n")
set, err := Load([]string{p})
if err != nil {
t.Fatalf("newer version should load, got %v", err)
}
if len(set.Warnings) == 0 {
t.Fatal("expected a warning about the newer version")
}
if err := set.Validate(); err != nil {
t.Fatalf("validate: %v", err)
}
}
func TestLoadAcceptsOmittedVersion(t *testing.T) {
dir := t.TempDir()
p := filepath.Join(dir, "relspec.yml")
write(t, p, "jobs:\n a:\n command: convert\n"+
" inputs:\n - path: a.dbml\n format: dbml\n"+
" output:\n format: json\n path: out.json\n")
set, err := Load([]string{p})
if err != nil {
t.Fatalf("omitted version should load, got %v", err)
}
if len(set.Warnings) != 0 {
t.Fatalf("omitted version should not warn, got %v", set.Warnings)
}
}
func TestLoadStillRejectsUnknownFieldsAtCurrentVersion(t *testing.T) {
dir := t.TempDir()
p := filepath.Join(dir, "relspec.yml")
write(t, p, "version: 1\njobs:\n a:\n command: convert\n bogus: true\n")
if _, err := Load([]string{p}); err == nil {
t.Fatal("expected unknown-field rejection at the current version")
}
}
func TestParseHumanSize(t *testing.T) {
cases := []struct {
in string
want int64
bad bool
}{
{"", 0, false},
{"512", 512, false},
{"512B", 512, false},
{"1KB", 1 << 10, false},
{"5MB", 5 << 20, false},
{"1gb", 1 << 30, false},
{" 2 MB ", 2 << 20, false},
{"nonsense", 0, true},
{"-1MB", 0, true},
}
for _, c := range cases {
got, err := parseHumanSize(c.in)
if c.bad {
if err == nil {
t.Errorf("parseHumanSize(%q): expected error", c.in)
}
continue
}
if err != nil {
t.Errorf("parseHumanSize(%q): %v", c.in, err)
continue
}
if got != c.want {
t.Errorf("parseHumanSize(%q) = %d, want %d", c.in, got, c.want)
}
}
}
func TestExpandEnv(t *testing.T) {
t.Setenv("RELSPEC_TEST_ROOT", "build/output")
got, err := ExpandEnv("${RELSPEC_TEST_ROOT}/schema-${RELSPEC_TEST_ROOT}")
if err != nil {
t.Fatal(err)
}
if want := "build/output/schema-build/output"; got != want {
t.Fatalf("ExpandEnv = %q, want %q", got, want)
}
if _, err := ExpandEnv("build/${RELSPEC_TEST_MISSING}"); err == nil {
t.Fatal("expected missing environment variable error")
}
}
@@ -239,6 +323,135 @@ func TestShippedExampleIsValid(t *testing.T) {
}
}
func TestFromJobWiring(t *testing.T) {
content := "version: 1\njobs:\n" +
" producer:\n command: convert\n" +
" inputs:\n - path: a.dbml\n format: dbml\n" +
" output:\n format: json\n path: build/schema.json\n" +
" consumer:\n command: convert\n" +
" inputs:\n - from_job: producer\n" +
" output:\n format: yaml\n path: build/schema.yaml\n"
set := loadOne(t, content)
if err := set.Validate(); err != nil {
t.Fatalf("validate: %v", err)
}
plan, err := set.Plan("consumer", true)
if err != nil {
t.Fatal(err)
}
if len(plan) != 2 || plan[0].Name != "producer" || plan[1].Name != "consumer" {
t.Fatalf("plan = %v, want [producer consumer]", plan)
}
}
func TestFromJobRejectsNonProducer(t *testing.T) {
content := "version: 1\njobs:\n" +
" lister:\n command: scripts-list\n script_dirs: [migrations]\n" +
" consumer:\n command: convert\n" +
" inputs:\n - from_job: lister\n" +
" output:\n format: yaml\n path: out.yaml\n"
set := loadOne(t, content)
if err := set.Validate(); err == nil || !strings.Contains(err.Error(), "from_job") {
t.Fatalf("want from_job producer error, got %v", err)
}
}
func TestFromJobRejectsUnknownJob(t *testing.T) {
content := "version: 1\njobs:\n" +
" consumer:\n command: convert\n" +
" inputs:\n - from_job: ghost\n" +
" output:\n format: yaml\n path: out.yaml\n"
set := loadOne(t, content)
if err := set.Validate(); err == nil || !strings.Contains(err.Error(), "unknown job") {
t.Fatalf("want unknown job error, got %v", err)
}
}
func TestFromJobCycleDetected(t *testing.T) {
content := "version: 1\njobs:\n" +
" a:\n command: convert\n" +
" inputs:\n - from_job: b\n" +
" output:\n format: json\n path: a.json\n" +
" b:\n command: convert\n" +
" inputs:\n - from_job: a\n" +
" output:\n format: json\n path: b.json\n"
set := loadOne(t, content)
if err := set.Validate(); err == nil || !strings.Contains(err.Error(), "cycle") {
t.Fatalf("want cycle error, got %v", err)
}
}
func TestSplitJobValidation(t *testing.T) {
set := loadOne(t, "version: 1\njobs:\n s:\n command: split\n"+
" inputs:\n - path: a.dbml\n format: dbml\n")
if err := set.Validate(); err == nil || !strings.Contains(err.Error(), "missing output") {
t.Fatalf("want missing output, got %v", err)
}
set = loadOne(t, "version: 1\njobs:\n s:\n command: split\n"+
" inputs:\n - path: a.dbml\n format: dbml\n"+
" select:\n tables: [users]\n"+
" output:\n format: json\n path: out.json\n")
if err := set.Validate(); err != nil {
t.Fatalf("expected valid split job, got %v", err)
}
}
func TestInspectJobValidation(t *testing.T) {
set := loadOne(t, "version: 1\njobs:\n i:\n command: inspect\n"+
" inputs:\n - path: a.dbml\n format: dbml\n")
if err := set.Validate(); err == nil || !strings.Contains(err.Error(), "report") {
t.Fatalf("want report required, got %v", err)
}
set = loadOne(t, "version: 1\njobs:\n i:\n command: inspect\n"+
" inputs:\n - path: a.dbml\n format: dbml\n"+
" report:\n format: json\n path: build/report.json\n")
if err := set.Validate(); err != nil {
t.Fatalf("expected valid inspect job, got %v", err)
}
}
func TestDiffJobValidation(t *testing.T) {
set := loadOne(t, "version: 1\njobs:\n d:\n command: diff\n"+
" inputs:\n - path: a.dbml\n format: dbml\n"+
" report:\n format: summary\n")
if err := set.Validate(); err == nil || !strings.Contains(err.Error(), "exactly 2 inputs") {
t.Fatalf("want exactly 2 inputs, got %v", err)
}
set = loadOne(t, "version: 1\njobs:\n d:\n command: diff\n"+
" inputs:\n - path: a.dbml\n format: dbml\n"+
" - path: b.dbml\n format: dbml\n"+
" report:\n format: summary\n")
if err := set.Validate(); err != nil {
t.Fatalf("expected valid diff job, got %v", err)
}
}
func TestScriptsExecValidation(t *testing.T) {
set := loadOne(t, "version: 1\njobs:\n x:\n command: scripts-exec\n"+
" script_dirs: [migrations]\n")
if err := set.Validate(); err == nil || !strings.Contains(err.Error(), "conn_env") {
t.Fatalf("want output.conn_env required, got %v", err)
}
set = loadOne(t, "version: 1\njobs:\n x:\n command: scripts-exec\n"+
" script_dirs: [migrations]\n"+
" output:\n conn_env: TARGET_DB_URL\n")
if err := set.Validate(); err != nil {
t.Fatalf("expected valid scripts-exec job, got %v", err)
}
}
func TestSafeJoinRejectsSymlinkEscape(t *testing.T) {
root := t.TempDir()
outside := t.TempDir()
link := filepath.Join(root, "link")
if err := os.Symlink(outside, link); err != nil {
t.Skipf("symlink not supported: %v", err)
}
if _, err := SafeJoin(root, "link/x.sql"); err == nil {
t.Fatal("expected rejection of a path escaping via a symlink")
}
}
func TestScriptsListValidation(t *testing.T) {
set := loadOne(t, "version: 1\njobs:\n s:\n command: scripts-list\n")
if err := set.Validate(); err == nil || !strings.Contains(err.Error(), "script_dir") {
+39
View File
@@ -91,6 +91,45 @@ func (r *MergeResult) merge(target, source *models.Database, opts *MergeOptions)
if !opts.SkipDomains {
r.mergeDomains(target, source)
}
mergeDatabaseMetadata(target, source)
}
// mergeDatabaseMetadata adds missing metadata keys and unions []string values,
// so per-file reader state (e.g. pending DBML commented refs) survives a merge.
func mergeDatabaseMetadata(target, source *models.Database) {
if len(source.Metadata) == 0 {
return
}
if target.Metadata == nil {
target.Metadata = make(map[string]any, len(source.Metadata))
}
for key, srcVal := range source.Metadata {
tgtVal, exists := target.Metadata[key]
if !exists {
if list, ok := srcVal.([]string); ok {
srcVal = append([]string(nil), list...)
}
target.Metadata[key] = srcVal
continue
}
tgtList, tgtOK := tgtVal.([]string)
srcList, srcOK := srcVal.([]string)
if !tgtOK || !srcOK {
continue
}
seen := make(map[string]bool, len(tgtList))
for _, v := range tgtList {
seen[v] = true
}
for _, v := range srcList {
if !seen[v] {
tgtList = append(tgtList, v)
seen[v] = true
}
}
target.Metadata[key] = tgtList
}
}
func (r *MergeResult) mergeSchemaContents(target, source *models.Schema, opts *MergeOptions) {
+29
View File
@@ -721,3 +721,32 @@ func TestComplexMerge(t *testing.T) {
t.Error("Expected ukey_users_guid constraint to exist")
}
}
func TestMergeDatabases_Metadata(t *testing.T) {
target := &models.Database{Metadata: map[string]any{
"refs": []string{"a", "b"},
"name": "target",
}}
source := &models.Database{Metadata: map[string]any{
"refs": []string{"b", "c"},
"name": "source",
"extra": []string{"x"},
}}
MergeDatabases(target, source, nil)
if got := target.Metadata["refs"].([]string); strings.Join(got, ",") != "a,b,c" {
t.Errorf("refs = %v, want [a b c]", got)
}
if got := target.Metadata["name"]; got != "target" {
t.Errorf("name = %v, want target (existing scalar keys are kept)", got)
}
extra := target.Metadata["extra"].([]string)
if strings.Join(extra, ",") != "x" {
t.Errorf("extra = %v, want [x]", extra)
}
source.Metadata["extra"].([]string)[0] = "changed"
if extra[0] != "x" {
t.Error("copied slice must not alias the source")
}
}
+22 -17
View File
@@ -20,6 +20,7 @@ const (
PostgresqlDatabaseType DatabaseType = "pgsql" // PostgreSQL database
MSSQLDatabaseType DatabaseType = "mssql" // Microsoft SQL Server database
SqlLiteDatabaseType DatabaseType = "sqlite" // SQLite database
MySQLDatabaseType DatabaseType = "mysql" // MySQL/MariaDB database
)
// Database represents the complete database schema
@@ -227,23 +228,27 @@ func (d *Sequence) SQLName() string {
// Column represents a table column
type Column struct {
Name string `json:"name" yaml:"name" xml:"name"`
Description string `json:"description,omitempty" yaml:"description,omitempty" xml:"description,omitempty"`
Table string `json:"table" yaml:"table" xml:"table"`
Schema string `json:"schema" yaml:"schema" xml:"schema"`
Type string `json:"type" yaml:"type" xml:"type"`
Length int `json:"length,omitempty" yaml:"length,omitempty" xml:"length,omitempty"`
Precision int `json:"precision,omitempty" yaml:"precision,omitempty" xml:"precision,omitempty"`
Scale int `json:"scale,omitempty" yaml:"scale,omitempty" xml:"scale,omitempty"`
NotNull bool `json:"not_null" yaml:"not_null" xml:"not_null"`
Default any `json:"default,omitempty" yaml:"default,omitempty" xml:"default,omitempty"`
AutoIncrement bool `json:"auto_increment" yaml:"auto_increment" xml:"auto_increment"`
IsPrimaryKey bool `json:"is_primary_key" yaml:"is_primary_key" xml:"is_primary_key"`
Comment string `json:"comment,omitempty" yaml:"comment,omitempty" xml:"comment,omitempty"`
Collation string `json:"collation,omitempty" yaml:"collation,omitempty" xml:"collation,omitempty"`
Metadata map[string]any `json:"metadata,omitempty" yaml:"metadata,omitempty" xml:"-"`
Sequence uint `json:"sequence,omitempty" yaml:"sequence,omitempty" xml:"sequence,omitempty"`
GUID string `json:"guid" yaml:"guid" xml:"guid"`
Name string `json:"name" yaml:"name" xml:"name"`
Description string `json:"description,omitempty" yaml:"description,omitempty" xml:"description,omitempty"`
Table string `json:"table" yaml:"table" xml:"table"`
Schema string `json:"schema" yaml:"schema" xml:"schema"`
Type string `json:"type" yaml:"type" xml:"type"`
Length int `json:"length,omitempty" yaml:"length,omitempty" xml:"length,omitempty"`
Precision int `json:"precision,omitempty" yaml:"precision,omitempty" xml:"precision,omitempty"`
Scale int `json:"scale,omitempty" yaml:"scale,omitempty" xml:"scale,omitempty"`
NotNull bool `json:"not_null" yaml:"not_null" xml:"not_null"`
Default any `json:"default,omitempty" yaml:"default,omitempty" xml:"default,omitempty"`
AutoIncrement bool `json:"auto_increment" yaml:"auto_increment" xml:"auto_increment"`
IsPrimaryKey bool `json:"is_primary_key" yaml:"is_primary_key" xml:"is_primary_key"`
Comment string `json:"comment,omitempty" yaml:"comment,omitempty" xml:"comment,omitempty"`
Collation string `json:"collation,omitempty" yaml:"collation,omitempty" xml:"collation,omitempty"`
Metadata map[string]any `json:"metadata,omitempty" yaml:"metadata,omitempty" xml:"-"`
Sequence uint `json:"sequence,omitempty" yaml:"sequence,omitempty" xml:"sequence,omitempty"`
GUID string `json:"guid" yaml:"guid" xml:"guid"`
Generated bool `json:"generated,omitempty" yaml:"generated,omitempty" xml:"generated,omitempty"`
GenerationExpression string `json:"generation_expression,omitempty" yaml:"generation_expression,omitempty" xml:"generation_expression,omitempty"`
Identity bool `json:"identity,omitempty" yaml:"identity,omitempty" xml:"identity,omitempty"`
IdentityGeneration string `json:"identity_generation,omitempty" yaml:"identity_generation,omitempty" xml:"identity_generation,omitempty"` // "ALWAYS" or "BY DEFAULT"
}
// SQLName returns the column name in lowercase for SQL compatibility.
+2
View File
@@ -64,6 +64,8 @@ The reader recognizes the following Bun struct tags:
- `autoincrement` - Auto-increment column
- `default` - Default value
- `unique` - Unique constraint
- `generated` - Generated column marker (`Generated`; expression is not in the tag)
- `identity` - Identity column marker (`Identity`, `ALWAYS`)
- `rel` - Relationship definition
## Example Bun Model
+48
View File
@@ -0,0 +1,48 @@
package bun
import (
"os"
"path/filepath"
"testing"
"git.warky.dev/wdevs/relspecgo/pkg/readers"
)
func TestReader_GeneratedAndIdentityMarkers(t *testing.T) {
src := `package models
import "github.com/uptrace/bun"
type Person struct {
bun.BaseModel ` + "`bun:\"table:people,alias:p\"`" + `
ID int64 ` + "`bun:\"id,pk,type:bigint,autoincrement,identity\"`" + `
Seq int64 ` + "`bun:\"seq,type:bigint,scanonly,identity,notnull\"`" + `
FullName string ` + "`bun:\"full_name,type:text,scanonly,generated,nullzero\"`" + `
First string ` + "`bun:\"first,type:text,nullzero\"`" + `
}
`
path := filepath.Join(t.TempDir(), "person.go")
if err := os.WriteFile(path, []byte(src), 0o644); err != nil {
t.Fatal(err)
}
db, err := NewReader(&readers.ReaderOptions{FilePath: path}).ReadDatabase()
if err != nil {
t.Fatalf("ReadDatabase() error = %v", err)
}
cols := db.Schemas[0].Tables[0].Columns
if !cols["id"].Identity || cols["id"].IdentityGeneration != "ALWAYS" {
t.Errorf("id: identity = %v/%q, want true/ALWAYS", cols["id"].Identity, cols["id"].IdentityGeneration)
}
if !cols["seq"].Identity || cols["seq"].IdentityGeneration != "ALWAYS" {
t.Errorf("seq: identity = %v/%q, want true/ALWAYS", cols["seq"].Identity, cols["seq"].IdentityGeneration)
}
if !cols["full_name"].Generated {
t.Error("full_name should be generated")
}
if cols["first"].Generated || cols["first"].Identity {
t.Error("first must be an ordinary column")
}
}
+7
View File
@@ -659,6 +659,13 @@ func (r *Reader) parseColumn(fieldName string, fieldType ast.Expr, tag string, s
hasExplicitNullableMarker = true
case "autoincrement":
column.AutoIncrement = true
case "generated":
// GENERATED ... STORED marker written by the Bun writer; the expression is not in the tag
column.Generated = true
case "identity":
// GENERATED ALWAYS AS IDENTITY marker written by the Bun writer
column.Identity = true
column.IdentityGeneration = "ALWAYS"
case "default":
// Default value from Bun tag (e.g., default:gen_random_uuid())
column.Default = value
+18
View File
@@ -90,11 +90,28 @@ Ref: posts.user_id > users.id [delete: cascade]
- Default values (`default`)
- Inline references (`ref`)
- Standalone `Ref` blocks
- Commented cross-file refs (`// Ref:` — see below)
- Indexes and composite indexes
- Table notes and column notes
- Enums
- Dialect directives (`@postgres:` / `@sqlite:` — see below)
## Commented cross-file refs
`// Ref:` / `// ref:` lines (ignored by dbdiagram) become FKs + relationships once both ends are loaded.
| Rule | Behaviour |
|---|---|
| When | Single file / directory: end of read. `--from-list`, `merge`, jobs: after all inputs are combined |
| Match | `schema.table.column` on both sides, case-insensitive |
| Operators | `>`, `<`, `-` (parsed like `Ref:`) |
| Duplicate of an FK on the same columns | Skipped silently |
| Target not loaded | Kept pending; skipped with a warning on the final pass |
| Column type mismatch | Warning, FK still created (`serial`≈`integer`, `bigserial`≈`bigint`) |
| Pending state | `Database.Metadata["dbml.commented_refs"]` (`[]string`) |
API: `dbml.ResolveCommentedRefs(db, final bool) []string` returns warnings.
## Dialect directives
Lines of the form `@<namespace>[(<column>)]: <args>` embed database-specific
@@ -140,6 +157,7 @@ grammar and the supported-directive matrix.
## Notes
- Column notes `GENERATED ALWAYS AS (expr) STORED` and `GENERATED ALWAYS|BY DEFAULT AS IDENTITY` set `Generated`/`GenerationExpression` and `Identity`/`IdentityGeneration`; they are not kept as comments
- DBML is designed for database documentation and diagramming
- Schema name defaults to `public`
- Relationship cardinality is preserved
+218
View File
@@ -0,0 +1,218 @@
package dbml
import (
"fmt"
"regexp"
"strings"
"git.warky.dev/wdevs/relspecgo/pkg/models"
)
// CommentedRefsMetadataKey is the Database.Metadata key holding commented
// `// Ref:` lines not yet resolved against the model ([]string).
const CommentedRefsMetadataKey = "dbml.commented_refs"
// commentedRefRegex matches `// Ref: ...` and `// ref: ...`.
var commentedRefRegex = regexp.MustCompile(`^//\s*[Rr]ef\s*:\s*(.+)$`)
// commentedRef returns the ref body of a trimmed `// Ref:` comment line.
func commentedRef(line string) (string, bool) {
m := commentedRefRegex.FindStringSubmatch(line)
if m == nil {
return "", false
}
ref := strings.TrimSpace(m[1])
return ref, ref != ""
}
// PendingCommentedRefs returns the commented refs not yet resolved.
func PendingCommentedRefs(db *models.Database) []string {
if db == nil || db.Metadata == nil {
return nil
}
refs, _ := db.Metadata[CommentedRefsMetadataKey].([]string)
return refs
}
func setPendingCommentedRefs(db *models.Database, refs []string) {
if len(refs) == 0 {
if db.Metadata != nil {
delete(db.Metadata, CommentedRefsMetadataKey)
}
return
}
if db.Metadata == nil {
db.Metadata = make(map[string]any)
}
db.Metadata[CommentedRefsMetadataKey] = refs
}
// addPendingCommentedRef queues a ref, skipping exact repeats.
func addPendingCommentedRef(db *models.Database, ref string) {
refs := PendingCommentedRefs(db)
for _, existing := range refs {
if existing == ref {
return
}
}
setPendingCommentedRefs(db, append(refs, ref))
}
// ResolveCommentedRefs turns pending commented refs into foreign keys and
// relationships when both ends (schema.table.column) exist in db. A ref that
// matches an existing FK on the same columns is dropped as a duplicate.
// Unmatched refs stay pending unless final is set, in which case they are
// dropped with a warning. Returns human-readable warnings.
func ResolveCommentedRefs(db *models.Database, final bool) []string {
refs := PendingCommentedRefs(db)
if len(refs) == 0 {
return nil
}
var warnings []string
var pending []string
parser := &Reader{}
for _, ref := range refs {
fk := parser.parseRef(ref)
if fk == nil || len(fk.Columns) == 0 || len(fk.Columns) != len(fk.ReferencedColumns) {
warnings = append(warnings, fmt.Sprintf("skipping commented ref %q: cannot parse", ref))
continue
}
srcTable, srcCols, srcMissing := lookupColumns(db, fk.Schema, fk.Table, fk.Columns)
dstTable, dstCols, dstMissing := lookupColumns(db, fk.ReferencedSchema, fk.ReferencedTable, fk.ReferencedColumns)
if srcMissing != "" || dstMissing != "" {
if final {
missing := srcMissing
if missing == "" {
missing = dstMissing
}
warnings = append(warnings, fmt.Sprintf("skipping commented ref %q: %s not found", ref, missing))
} else {
pending = append(pending, ref)
}
continue
}
fk.Schema, fk.Table, fk.Columns = srcTable.Schema, srcTable.Name, columnNames(srcCols)
fk.ReferencedSchema, fk.ReferencedTable, fk.ReferencedColumns = dstTable.Schema, dstTable.Name, columnNames(dstCols)
if hasFKOnColumns(srcTable, fk.Columns) {
continue // already declared by an uncommented Ref or inline ref
}
if _, taken := srcTable.Constraints[fk.Name]; taken {
warnings = append(warnings, fmt.Sprintf("skipping commented ref %q: constraint %s already exists", ref, fk.Name))
continue
}
for i := range srcCols {
if !compatibleFKTypes(srcCols[i].Type, dstCols[i].Type) {
warnings = append(warnings, fmt.Sprintf("commented ref %q: type mismatch %s.%s.%s (%s) -> %s.%s.%s (%s)",
ref, srcTable.Schema, srcTable.Name, srcCols[i].Name, srcCols[i].Type,
dstTable.Schema, dstTable.Name, dstCols[i].Name, dstCols[i].Type))
}
}
if srcTable.Constraints == nil {
srcTable.Constraints = make(map[string]*models.Constraint)
}
srcTable.Constraints[fk.Name] = fk
addFKRelationship(srcTable, fk)
}
setPendingCommentedRefs(db, pending)
return warnings
}
// lookupColumns finds a table and its columns, case-insensitively. missing
// names the first object not found, or is empty.
func lookupColumns(db *models.Database, schemaName, tableName string, cols []string) (*models.Table, []*models.Column, string) {
qualified := schemaName + "." + tableName
var table *models.Table
for _, schema := range db.Schemas {
if !strings.EqualFold(schema.Name, schemaName) {
continue
}
for _, t := range schema.Tables {
if strings.EqualFold(t.Name, tableName) {
table = t
break
}
}
}
if table == nil {
return nil, nil, "table " + qualified
}
found := make([]*models.Column, 0, len(cols))
for _, name := range cols {
col := table.Columns[name]
if col == nil {
for _, c := range table.Columns {
if strings.EqualFold(c.Name, name) {
col = c
break
}
}
}
if col == nil {
return nil, nil, "column " + qualified + "." + name
}
found = append(found, col)
}
return table, found, ""
}
func columnNames(cols []*models.Column) []string {
names := make([]string, len(cols))
for i, c := range cols {
names[i] = c.Name
}
return names
}
// hasFKOnColumns reports whether table already has a foreign key over cols.
func hasFKOnColumns(table *models.Table, cols []string) bool {
for _, c := range table.Constraints {
if c.Type != models.ForeignKeyConstraint || len(c.Columns) != len(cols) {
continue
}
same := true
for i := range cols {
if !strings.EqualFold(c.Columns[i], cols[i]) {
same = false
break
}
}
if same {
return true
}
}
return false
}
// fkTypeAliases maps serial and alias spellings to their storage type.
var fkTypeAliases = map[string]string{
"smallserial": "smallint", "serial2": "smallint", "int2": "smallint",
"serial": "integer", "serial4": "integer", "int": "integer", "int4": "integer",
"bigserial": "bigint", "serial8": "bigint", "int8": "bigint",
}
// compatibleFKTypes compares column types ignoring case, length and serial
// vs. integer spelling. Unknown (empty) types are treated as compatible.
func compatibleFKTypes(a, b string) bool {
na, nb := normalizeFKType(a), normalizeFKType(b)
return na == "" || nb == "" || na == nb
}
func normalizeFKType(t string) string {
t = strings.ToLower(strings.TrimSpace(t))
if i := strings.Index(t, "("); i >= 0 {
t = strings.TrimSpace(t[:i])
}
if alias, ok := fkTypeAliases[t]; ok {
return alias
}
return t
}
+260
View File
@@ -0,0 +1,260 @@
package dbml
import (
"os"
"path/filepath"
"strings"
"testing"
"git.warky.dev/wdevs/relspecgo/pkg/models"
"git.warky.dev/wdevs/relspecgo/pkg/readers"
)
func readDBMLString(t *testing.T, content string) *models.Database {
t.Helper()
path := filepath.Join(t.TempDir(), "in.dbml")
if err := os.WriteFile(path, []byte(content), 0o644); err != nil {
t.Fatalf("write fixture: %v", err)
}
db, err := NewReader(&readers.ReaderOptions{FilePath: path}).ReadDatabase()
if err != nil {
t.Fatalf("ReadDatabase() error = %v", err)
}
return db
}
func findTable(db *models.Database, schema, table string) *models.Table {
for _, s := range db.Schemas {
if s.Name != schema {
continue
}
for _, t := range s.Tables {
if t.Name == table {
return t
}
}
}
return nil
}
func fkOn(table *models.Table, col string) *models.Constraint {
for _, c := range table.Constraints {
if c.Type == models.ForeignKeyConstraint && len(c.Columns) == 1 && c.Columns[0] == col {
return c
}
}
return nil
}
func relFor(table *models.Table, fkName string) *models.Relationship {
for _, r := range table.Relationships {
if r.ForeignKey == fkName {
return r
}
}
return nil
}
func TestCommentedRef(t *testing.T) {
tests := []struct {
line string
want string
ok bool
}{
{`// Ref: a.b.c > d.e.f`, `a.b.c > d.e.f`, true},
{`// ref: a.b.c - d.e.f`, `a.b.c - d.e.f`, true},
{`//Ref:a.b.c > d.e.f`, `a.b.c > d.e.f`, true},
{`// Reference notes`, "", false},
{`// see Ref: a.b.c > d.e.f`, "", false},
{`// Ref:`, "", false},
}
for _, tt := range tests {
got, ok := commentedRef(tt.line)
if ok != tt.ok || got != tt.want {
t.Errorf("commentedRef(%q) = (%q, %v), want (%q, %v)", tt.line, got, ok, tt.want, tt.ok)
}
}
}
// A commented ref whose tables are in the same file resolves on read.
func TestReader_CommentedRefSameFile(t *testing.T) {
db := readDBMLString(t, `Table "org"."department" {
"id_department" bigserial [pk]
}
Table "entity"."employee" {
"id_employee" bigserial [pk]
"rid_department" bigint
}
// Ref: "entity"."employee"."rid_department" > "org"."department"."id_department" [delete: restrict, update: restrict]
`)
emp := findTable(db, "entity", "employee")
fk := fkOn(emp, "rid_department")
if fk == nil {
t.Fatal("expected FK on rid_department")
}
if fk.ReferencedSchema != "org" || fk.ReferencedTable != "department" || fk.ReferencedColumns[0] != "id_department" {
t.Errorf("FK target = %s.%s.%v", fk.ReferencedSchema, fk.ReferencedTable, fk.ReferencedColumns)
}
if fk.OnDelete != "restrict" || fk.OnUpdate != "restrict" {
t.Errorf("FK actions = %q/%q, want restrict/restrict", fk.OnDelete, fk.OnUpdate)
}
if relFor(emp, fk.Name) == nil {
t.Error("expected relationship for FK")
}
if refs := PendingCommentedRefs(db); len(refs) != 0 {
t.Errorf("pending = %v, want none", refs)
}
}
// A cross-file commented ref stays pending after a single-file read.
func TestReader_CommentedRefCrossFilePending(t *testing.T) {
db := readDBMLString(t, `Table "entity"."employee" {
"id_employee" bigserial [pk]
"rid_department" bigint
}
// Ref: "entity"."employee"."rid_department" > "org"."department"."id_department"
`)
if fk := fkOn(findTable(db, "entity", "employee"), "rid_department"); fk != nil {
t.Fatal("FK must not resolve without the target table")
}
if refs := PendingCommentedRefs(db); len(refs) != 1 {
t.Fatalf("pending = %v, want 1", refs)
}
}
// Directory reads resolve commented refs after all files are merged.
func TestReader_CommentedRefDirectory(t *testing.T) {
db, err := NewReader(&readers.ReaderOptions{
FilePath: filepath.Join("..", "..", "..", "tests", "assets", "dbml", "multifile"),
}).ReadDatabase()
if err != nil {
t.Fatalf("ReadDatabase() error = %v", err)
}
fk := fkOn(findTable(db, "public", "posts"), "user_id")
if fk == nil {
t.Fatal("expected FK posts.user_id from 9_refs.dbml commented ref")
}
if fk.ReferencedTable != "users" || fk.OnDelete != "CASCADE" {
t.Errorf("FK = %s ondelete %s, want users ondelete CASCADE", fk.ReferencedTable, fk.OnDelete)
}
}
func crossFileDB(t *testing.T, refs ...string) *models.Database {
t.Helper()
db := readDBMLString(t, `Table "org"."department" {
"id_department" bigserial [pk]
}
Table "entity"."employee" {
"id_employee" bigserial [pk]
"rid_department" bigint
"rid_manager" integer
"rid_team" bigint
}
`)
setPendingCommentedRefs(db, refs)
return db
}
func TestResolveCommentedRefs(t *testing.T) {
t.Run("lowercase ref and one-to-one", func(t *testing.T) {
db := crossFileDB(t,
`"entity"."employee"."rid_department" > "org"."department"."id_department"`,
`entity.employee.rid_team - org.department.id_department`,
)
if w := ResolveCommentedRefs(db, true); len(w) != 0 {
t.Errorf("warnings = %v, want none", w)
}
emp := findTable(db, "entity", "employee")
for _, col := range []string{"rid_department", "rid_team"} {
fk := fkOn(emp, col)
if fk == nil {
t.Fatalf("expected FK on %s", col)
}
if relFor(emp, fk.Name) == nil {
t.Errorf("expected relationship for %s", fk.Name)
}
}
if len(emp.Relationships) != 2 {
t.Errorf("relationships = %d, want 2 (same target must not overwrite)", len(emp.Relationships))
}
})
t.Run("type mismatch warns but resolves", func(t *testing.T) {
db := crossFileDB(t, `entity.employee.rid_manager > org.department.id_department`)
w := ResolveCommentedRefs(db, true)
if len(w) != 1 || !strings.Contains(w[0], "type mismatch") {
t.Errorf("warnings = %v, want one type mismatch", w)
}
if fkOn(findTable(db, "entity", "employee"), "rid_manager") == nil {
t.Error("expected FK on rid_manager")
}
})
t.Run("missing target stays pending until final", func(t *testing.T) {
db := crossFileDB(t, `entity.employee.rid_team > hr.team.id_team`)
if w := ResolveCommentedRefs(db, false); len(w) != 0 {
t.Errorf("non-final warnings = %v, want none", w)
}
if len(PendingCommentedRefs(db)) != 1 {
t.Fatal("ref should stay pending")
}
w := ResolveCommentedRefs(db, true)
if len(w) != 1 || !strings.Contains(w[0], "table hr.team not found") {
t.Errorf("final warnings = %v, want missing table", w)
}
if len(PendingCommentedRefs(db)) != 0 {
t.Error("final pass must clear pending refs")
}
})
t.Run("missing column", func(t *testing.T) {
db := crossFileDB(t, `entity.employee.rid_nope > org.department.id_department`)
w := ResolveCommentedRefs(db, true)
if len(w) != 1 || !strings.Contains(w[0], "column entity.employee.rid_nope not found") {
t.Errorf("warnings = %v, want missing column", w)
}
})
t.Run("deduplicates against uncommented ref", func(t *testing.T) {
db := readDBMLString(t, `Table "org"."department" {
"id_department" bigserial [pk]
}
Table "entity"."employee" {
"id_employee" bigserial [pk]
"rid_department" bigint
}
Ref: "entity"."employee"."rid_department" > "org"."department"."id_department"
// Ref: "entity"."employee"."rid_department" > "org"."department"."id_department"
`)
emp := findTable(db, "entity", "employee")
count := 0
for _, c := range emp.Constraints {
if c.Type == models.ForeignKeyConstraint {
count++
}
}
if count != 1 || len(emp.Relationships) != 1 {
t.Errorf("FKs = %d, relationships = %d, want 1 and 1", count, len(emp.Relationships))
}
})
}
func TestCompatibleFKTypes(t *testing.T) {
tests := []struct {
a, b string
want bool
}{
{"bigint", "bigserial", true},
{"integer", "serial", true},
{"INT4", "integer", true},
{"varchar(10)", "varchar(20)", true},
{"bigint", "serial", false},
{"uuid", "bigint", false},
{"", "bigint", true},
}
for _, tt := range tests {
if got := compatibleFKTypes(tt.a, tt.b); got != tt.want {
t.Errorf("compatibleFKTypes(%q, %q) = %v, want %v", tt.a, tt.b, got, tt.want)
}
}
}
+38
View File
@@ -0,0 +1,38 @@
package dbml
import (
"regexp"
"strings"
"git.warky.dev/wdevs/relspecgo/pkg/models"
)
var (
generatedNoteRegex = regexp.MustCompile(`(?is)^GENERATED\s+ALWAYS\s+AS\s*\((.*)\)\s*STORED$`)
identityNoteRegex = regexp.MustCompile(`(?i)^GENERATED\s+(ALWAYS|BY\s+DEFAULT)\s+AS\s+IDENTITY$`)
)
// applyColumnMarkerNote interprets a column note written by the DBML writer for
// GENERATED ... STORED and GENERATED ... AS IDENTITY columns, setting the matching
// model fields. It returns true when the note was a marker, in which case it is not
// a user comment.
func applyColumnMarkerNote(column *models.Column, rawNote string) bool {
note := strings.TrimSpace(rawNote)
if len(note) >= 2 && (note[0] == '\'' || note[0] == '"') && note[len(note)-1] == note[0] {
note = note[1 : len(note)-1]
}
note = strings.ReplaceAll(strings.ReplaceAll(note, `\'`, `'`), `\\`, `\`)
note = strings.TrimSpace(note)
if m := generatedNoteRegex.FindStringSubmatch(note); m != nil {
column.Generated = true
column.GenerationExpression = strings.TrimSpace(m[1])
return true
}
if m := identityNoteRegex.FindStringSubmatch(note); m != nil {
column.Identity = true
column.IdentityGeneration = strings.Join(strings.Fields(strings.ToUpper(m[1])), " ")
return true
}
return false
}
+195 -24
View File
@@ -48,7 +48,22 @@ func (r *Reader) ReadDatabase() (*models.Database, error) {
return nil, fmt.Errorf("failed to read file: %w", err)
}
return r.parseDBML(string(content))
db, err := r.parseDBML(string(content))
if err != nil {
return nil, err
}
r.resolveCommentedRefs(db)
return db, nil
}
// resolveCommentedRefs resolves the commented refs whose tables are loaded.
// Unmatched refs stay pending for a later pass over a combined model.
func (r *Reader) resolveCommentedRefs(db *models.Database) {
for _, w := range ResolveCommentedRefs(db, false) {
if r.options.Progress != nil {
r.options.Progress("warning: " + w)
}
}
}
// ReadSchema reads and parses DBML input, returning a Schema model
@@ -125,6 +140,7 @@ func (r *Reader) readDirectoryDBML(dirPath string) (*models.Database, error) {
}
}
r.resolveCommentedRefs(db)
return db, nil
}
@@ -305,9 +321,39 @@ func sortDBMLFiles(files []string) []string {
// Merges: Columns (map), Constraints (map), Indexes (map), Relationships (map)
// Uses first non-empty Description
func mergeTable(baseTable, fileTable *models.Table) {
// Merge columns (map naturally merges - later keys overwrite)
for key, col := range fileTable.Columns {
baseTable.Columns[key] = col
// Merge columns. Each file numbers its own columns from 1, so a table split
// across files would otherwise end up with colliding Column.Sequence values
// and writers would fall back to alphabetical order. Re-base the incoming
// file's new columns after the highest sequence already present, preserving
// their in-file order. Columns that overwrite an existing key keep the
// original position.
var maxSeq uint
for _, col := range baseTable.Columns {
if col.Sequence > maxSeq {
maxSeq = col.Sequence
}
}
incoming := make([]*models.Column, 0, len(fileTable.Columns))
for _, col := range fileTable.Columns {
incoming = append(incoming, col)
}
sort.Slice(incoming, func(i, j int) bool {
if incoming[i].Sequence != incoming[j].Sequence {
return incoming[i].Sequence < incoming[j].Sequence
}
return incoming[i].Name < incoming[j].Name
})
var added uint
for _, col := range incoming {
if existing, ok := baseTable.Columns[col.Name]; ok {
col.Sequence = existing.Sequence
} else {
added++
col.Sequence = maxSeq + added
}
baseTable.Columns[col.Name] = col
}
// Merge constraints
@@ -410,6 +456,10 @@ func mergeDatabase(baseDB, fileDB *models.Database) {
// Merge domains
baseDB.Domains = append(baseDB.Domains, fileDB.Domains...)
for _, ref := range PendingCommentedRefs(fileDB) {
addPendingCommentedRef(baseDB, ref)
}
// Use first non-empty description
if baseDB.Description == "" && fileDB.Description != "" {
baseDB.Description = fileDB.Description
@@ -434,6 +484,9 @@ func (r *Reader) parseDBML(content string) (*models.Database, error) {
var currentSchema string
var inIndexes bool
var inTable bool
var inTableNote bool
var tableNoteLines []string
tableNoteStartLine := 0
var columnSeq uint
var lastIndex *models.Index // most recent index in the current Indexes block
lineNo := 0
@@ -443,10 +496,29 @@ func (r *Reader) parseDBML(content string) (*models.Database, error) {
for scanner.Scan() {
lineNo++
line := strings.TrimSpace(scanner.Text())
rawLine := scanner.Text()
line := strings.TrimSpace(rawLine)
// Skip empty lines and comments
// A table note can use DBML's triple-quoted form. Its contents must be
// consumed before normal parsing, otherwise each prose line is mistaken
// for a column declaration.
if inTableNote {
if line == "'''" {
setTableNote(currentTable, strings.TrimSpace(strings.Join(tableNoteLines, "\n")))
inTableNote = false
tableNoteLines = nil
continue
}
tableNoteLines = append(tableNoteLines, strings.TrimSpace(rawLine))
continue
}
// Skip empty lines and comments. A commented `// Ref:` is kept as a
// pending cross-file ref, resolved once every file is loaded.
if line == "" || strings.HasPrefix(line, "//") {
if ref, ok := commentedRef(line); ok {
addPendingCommentedRef(db, ref)
}
continue
}
@@ -533,17 +605,30 @@ func (r *Reader) parseDBML(content string) (*models.Database, error) {
index := r.parseIndex(line, currentTable.Name, currentSchema)
if index != nil {
currentTable.Indexes[index.Name] = index
// Keep a reused name under a unique map key so the duplicate is
// not silently dropped; the inspector reports it.
key := index.Name
for n := 2; currentTable.Indexes[key] != nil; n++ {
key = fmt.Sprintf("%s#%d", index.Name, n)
}
currentTable.Indexes[key] = index
lastIndex = index
}
continue
}
// Parse table note
if inTable && currentTable != nil && strings.HasPrefix(line, "Note:") {
note := strings.TrimPrefix(line, "Note:")
// Parse table note. DBML files in the wild use both `Note:` and
// `note:`, so accept either spelling.
if inTable && currentTable != nil && strings.HasPrefix(strings.ToLower(line), "note:") {
note := strings.TrimSpace(line[len("note:"):])
if strings.TrimSpace(note) == "'''" {
inTableNote = true
tableNoteLines = nil
tableNoteStartLine = lineNo
continue
}
note = strings.Trim(note, " '\"")
currentTable.Description = note
setTableNote(currentTable, note)
continue
}
@@ -580,6 +665,13 @@ func (r *Reader) parseDBML(content string) (*models.Database, error) {
}
}
if err := scanner.Err(); err != nil {
return nil, fmt.Errorf("failed to scan DBML: %w", err)
}
if inTableNote {
return nil, fmt.Errorf("dbml: line %d: unterminated triple-quoted table note", tableNoteStartLine)
}
// Assign pending constraints to their respective tables
for _, constraint := range pendingConstraints {
// Find the table this constraint belongs to
@@ -597,20 +689,12 @@ func (r *Reader) parseDBML(content string) (*models.Database, error) {
// for DBML refs so diffing equivalent schemas compares the same model.
for _, schema := range schemaMap {
for _, table := range schema.Tables {
for _, constraint := range table.Constraints {
for _, name := range sortedConstraintNames(table.Constraints) {
constraint := table.Constraints[name]
if constraint.Type != models.ForeignKeyConstraint {
continue
}
name := fmt.Sprintf("%s_to_%s", table.Name, constraint.ReferencedTable)
relationship := models.InitRelationship(name, models.OneToMany)
relationship.FromTable = table.Name
relationship.FromSchema = table.Schema
relationship.FromColumns = append([]string(nil), constraint.Columns...)
relationship.ToTable = constraint.ReferencedTable
relationship.ToSchema = constraint.ReferencedSchema
relationship.ToColumns = append([]string(nil), constraint.ReferencedColumns...)
relationship.ForeignKey = constraint.Name
table.Relationships[name] = relationship
addFKRelationship(table, constraint)
}
}
}
@@ -623,6 +707,53 @@ func (r *Reader) parseDBML(content string) (*models.Database, error) {
return db, nil
}
// sortedConstraintNames returns constraint keys in sorted order so derived
// relationship names do not depend on map iteration order.
func sortedConstraintNames(constraints map[string]*models.Constraint) []string {
names := make([]string, 0, len(constraints))
for name := range constraints {
names = append(names, name)
}
sort.Strings(names)
return names
}
// addFKRelationship derives the relationship for a foreign key, matching how
// the PostgreSQL reader models FKs.
func addFKRelationship(table *models.Table, constraint *models.Constraint) {
name := fmt.Sprintf("%s_to_%s", table.Name, constraint.ReferencedTable)
if existing, taken := table.Relationships[name]; taken && existing.ForeignKey != constraint.Name {
// A second FK to the same table must not overwrite the first.
name = fmt.Sprintf("%s_%s", name, strings.Join(constraint.Columns, "_"))
}
relationship := models.InitRelationship(name, models.OneToMany)
relationship.FromTable = table.Name
relationship.FromSchema = table.Schema
relationship.FromColumns = append([]string(nil), constraint.Columns...)
relationship.ToTable = constraint.ReferencedTable
relationship.ToSchema = constraint.ReferencedSchema
relationship.ToColumns = append([]string(nil), constraint.ReferencedColumns...)
relationship.ForeignKey = constraint.Name
if table.Relationships == nil {
table.Relationships = make(map[string]*models.Relationship)
}
table.Relationships[name] = relationship
}
// setTableNote preserves multiple table notes. The first maps to Description
// and the second to Comment, matching the model fields used by code writers.
func setTableNote(table *models.Table, note string) {
if table.Description == "" {
table.Description = note
return
}
if table.Comment == "" {
table.Comment = note
return
}
table.Comment += "\n" + note
}
// parseColumn parses a DBML column definition
func (r *Reader) parseColumn(line, tableName, schemaName string) (*models.Column, *models.Constraint) {
// Format: column_name type [attributes] // comment
@@ -640,7 +771,7 @@ func (r *Reader) parseColumn(line, tableName, schemaName string) (*models.Column
// Parse attributes in brackets
if attrs != "" {
attrList := strings.Split(attrs, ",")
attrList := splitColumnAttrs(attrs)
for _, attr := range attrList {
attr = strings.TrimSpace(attr)
@@ -673,7 +804,9 @@ func (r *Reader) parseColumn(line, tableName, schemaName string) (*models.Column
} else if strings.HasPrefix(attr, "note:") {
// Parse column note/comment
note := strings.TrimSpace(strings.TrimPrefix(attr, "note:"))
column.Comment = strings.Trim(note, "'\"")
if !applyColumnMarkerNote(column, note) {
column.Comment = strings.Trim(note, "'\"")
}
} else if strings.HasPrefix(attr, "ref:") {
// Parse inline reference
// DBML semantics depend on context:
@@ -723,6 +856,44 @@ func (r *Reader) parseColumn(line, tableName, schemaName string) (*models.Column
return column, constraint
}
// splitColumnAttrs splits a DBML attribute list on top-level commas. Notes and
// quoted defaults may contain commas of their own, which are part of the value
// rather than attribute separators.
func splitColumnAttrs(attrs string) []string {
var result []string
start := 0
var quote byte
escaped := false
for i := 0; i < len(attrs); i++ {
ch := attrs[i]
if quote != 0 {
if escaped {
escaped = false
continue
}
if ch == '\\' {
escaped = true
continue
}
if ch == quote {
quote = 0
}
continue
}
switch ch {
case '\'', '"', '`':
quote = ch
case ',':
result = append(result, attrs[start:i])
start = i + 1
}
}
return append(result, attrs[start:])
}
func splitInlineComment(line string) (content, inlineComment string) {
commentStart := strings.Index(line, "//")
if commentStart == -1 {
+88
View File
@@ -652,6 +652,22 @@ func TestReadDirectory_TableMerging(t *testing.T) {
if emailCol.Type != "varchar(255)" {
t.Errorf("Expected email type 'varchar(255)', got '%s'", emailCol.Type)
}
// Merged columns must keep declaration order (file 1: id, email; file 3:
// name, created_at) via strictly increasing, non-colliding Sequence values
// so downstream writers do not fall back to alphabetical order.
order := []string{"id", "email", "name", "created_at"}
var prev uint
for i, name := range order {
col := usersTable.Columns[name]
if col.Sequence == 0 {
t.Fatalf("column %q has zero Sequence after merge", name)
}
if i > 0 && col.Sequence <= prev {
t.Errorf("column %q Sequence %d not greater than previous %d (order not preserved across files)", name, col.Sequence, prev)
}
prev = col.Sequence
}
}
func TestReadDirectory_CommentedRefsLast(t *testing.T) {
@@ -1033,3 +1049,75 @@ func TestReader_ColumnPKOrderPreserved(t *testing.T) {
t.Errorf("expected snapshot_id (declared first) to have a lower Sequence than artifact_id, got %d >= %d", snapshotCol.Sequence, artifactCol.Sequence)
}
}
func TestReader_MultilineTableNote(t *testing.T) {
dbmlContent := "Table \"info\".\"city\" {\n" +
" \"id_city\" serial [pk, not null, increment]\n" +
" \"name\" text [not null, note: 'first, second, third']\n\n" +
" note: '''\n" +
" Cities and municipalities worldwide.\n\n" +
" SPATIAL:\n" +
" Proximity queries use a GiST index.\n" +
" '''\n" +
" Note: 'Short summary'\n" +
"}\n"
path := filepath.Join(t.TempDir(), "city.dbml")
if err := os.WriteFile(path, []byte(dbmlContent), 0o644); err != nil {
t.Fatalf("failed to write fixture: %v", err)
}
db, err := NewReader(&readers.ReaderOptions{FilePath: path}).ReadDatabase()
if err != nil {
t.Fatalf("ReadDatabase() error = %v", err)
}
table := db.Schemas[0].Tables[0]
if got, want := table.Description, "Cities and municipalities worldwide.\n\nSPATIAL:\nProximity queries use a GiST index."; got != want {
t.Errorf("table description = %q, want %q", got, want)
}
if got, want := table.Comment, "Short summary"; got != want {
t.Errorf("table comment = %q, want %q", got, want)
}
if got, want := len(table.Columns), 2; got != want {
t.Errorf("column count = %d, want %d; note body must not be parsed as columns", got, want)
}
if got, want := table.Columns["name"].Comment, "first, second, third"; got != want {
t.Errorf("column note = %q, want %q", got, want)
}
}
// A name reused inside one Indexes block must not silently drop the earlier
// index; both are kept so the inspector can report the duplicate.
func TestReader_DuplicateIndexNameInTableKept(t *testing.T) {
dbmlContent := `Table individual_actor {
id bigint [pk]
rid_actor bigint
kind text
Indexes {
(rid_actor) [name: 'idx_actor', unique]
(rid_actor, kind) [name: 'idx_actor']
}
}
`
dir := t.TempDir()
path := filepath.Join(dir, "dup_index.dbml")
if err := os.WriteFile(path, []byte(dbmlContent), 0o644); err != nil {
t.Fatalf("failed to write fixture: %v", err)
}
db, err := NewReader(&readers.ReaderOptions{FilePath: path}).ReadDatabase()
if err != nil {
t.Fatalf("ReadDatabase() error = %v", err)
}
table := db.Schemas[0].Tables[0]
if len(table.Indexes) != 2 {
t.Fatalf("expected 2 indexes, got %d: %v", len(table.Indexes), table.Indexes)
}
for key, idx := range table.Indexes {
if idx.Name != "idx_actor" {
t.Errorf("index %q: Name = %q, want idx_actor", key, idx.Name)
}
}
}
+1
View File
@@ -85,6 +85,7 @@ export const postsRelations = relations(posts, ({ one }) => ({
## Notes
- `.generatedAlwaysAs(sql`expr`)` sets `Generated` + `GenerationExpression`; `.generatedAlwaysAsIdentity()` / `.generatedByDefaultAsIdentity()` set `Identity` + `IdentityGeneration`
- Supports both PostgreSQL and MySQL Drizzle schemas
- Extracts relationship information from `relations` definitions
- Schema defaults to `public` for PostgreSQL
+47 -1
View File
@@ -495,9 +495,24 @@ func (r *Reader) parseColumnModifiers(line string, column *models.Column, table
}
}
// Check for .generatedAlwaysAsIdentity()
// Check for .generatedAlwaysAsIdentity() / .generatedByDefaultAsIdentity()
if strings.Contains(line, ".generatedAlwaysAsIdentity()") {
column.AutoIncrement = true
column.Identity = true
column.IdentityGeneration = "ALWAYS"
}
if strings.Contains(line, ".generatedByDefaultAsIdentity()") {
column.AutoIncrement = true
column.Identity = true
column.IdentityGeneration = "BY DEFAULT"
}
// Check for .generatedAlwaysAs(sql`expr`) (generated column)
if idx := strings.Index(line, ".generatedAlwaysAs("); idx != -1 {
if expr, ok := parseGeneratedAlwaysAs(line[idx+len(".generatedAlwaysAs("):]); ok {
column.Generated = true
column.GenerationExpression = expr
}
}
// Check for .references(() => otherTable.column)
@@ -615,3 +630,34 @@ func (r *Reader) varNameToTableName(varName string) string {
// For now, assume variable name matches table name
return varName
}
// parseGeneratedAlwaysAs extracts the SQL expression from the argument list of
// generatedAlwaysAs(...), which follows the opening parenthesis in rest. The expression is
// the contents of a sql`...` template (backticks inside it are escaped with a backslash)
// or, failing that, a quoted string.
func parseGeneratedAlwaysAs(rest string) (string, bool) {
rest = strings.TrimSpace(rest)
rest = strings.TrimPrefix(rest, "sql")
if rest == "" {
return "", false
}
quote := rest[0]
if quote != '`' && quote != '\'' && quote != '"' {
return "", false
}
var sb strings.Builder
for i := 1; i < len(rest); i++ {
ch := rest[i]
if ch == '\\' && i+1 < len(rest) {
i++
sb.WriteByte(rest[i])
continue
}
if ch == quote {
return strings.TrimSpace(sb.String()), true
}
sb.WriteByte(ch)
}
return "", false
}
+2
View File
@@ -78,6 +78,8 @@ The reader recognizes the following GORM struct tags:
- `not null` - NOT NULL constraint
- `autoIncrement` - Auto-increment column
- `default` - Default value
- `generated` - Generated column marker (`Generated`; expression is not in the tag)
- `identity` - Identity column marker (`Identity`, `ALWAYS`)
- `size` - Column size/length
- `index` - Create index
- `uniqueIndex` - Create unique index
+48
View File
@@ -0,0 +1,48 @@
package gorm
import (
"os"
"path/filepath"
"testing"
"git.warky.dev/wdevs/relspecgo/pkg/readers"
)
func TestReader_GeneratedAndIdentityMarkers(t *testing.T) {
src := `package models
type Person struct {
ID int64 ` + "`gorm:\"column:id;primaryKey;autoIncrement;type:bigint;identity\"`" + `
Seq int64 ` + "`gorm:\"column:seq;type:bigint;<-:false;identity;not null\"`" + `
FullName string ` + "`gorm:\"column:full_name;type:text;<-:false;generated\"`" + `
First string ` + "`gorm:\"column:first;type:text\"`" + `
}
func (Person) TableName() string {
return "people"
}
`
path := filepath.Join(t.TempDir(), "person.go")
if err := os.WriteFile(path, []byte(src), 0o644); err != nil {
t.Fatal(err)
}
db, err := NewReader(&readers.ReaderOptions{FilePath: path}).ReadDatabase()
if err != nil {
t.Fatalf("ReadDatabase() error = %v", err)
}
cols := db.Schemas[0].Tables[0].Columns
if !cols["id"].Identity || cols["id"].IdentityGeneration != "ALWAYS" {
t.Errorf("id: identity = %v/%q, want true/ALWAYS", cols["id"].Identity, cols["id"].IdentityGeneration)
}
if !cols["seq"].Identity {
t.Error("seq should be identity")
}
if !cols["full_name"].Generated {
t.Error("full_name should be generated")
}
if cols["first"].Generated || cols["first"].Identity {
t.Error("first must be an ordinary column")
}
}
+9
View File
@@ -726,6 +726,15 @@ func (r *Reader) parseColumn(fieldName string, fieldType ast.Expr, tag string, s
if _, ok := parts["autoincrement"]; ok {
column.AutoIncrement = true
}
if _, ok := parts["generated"]; ok {
// GENERATED ... STORED marker written by the GORM writer; the expression is not in the tag
column.Generated = true
}
if _, ok := parts["identity"]; ok {
// GENERATED ALWAYS AS IDENTITY marker written by the GORM writer
column.Identity = true
column.IdentityGeneration = "ALWAYS"
}
if def, ok := parts["default"]; ok {
// Default value from GORM tag (e.g., default:gen_random_uuid())
column.Default = def
+5
View File
@@ -25,6 +25,11 @@ sqlserver://user:pass@192.168.1.100:1433/production
sqlserver://localhost/testdb?encrypt=disable
```
## Computed and Identity Columns
- `sys.computed_columns` -> `Generated` + `GenerationExpression` (outer parentheses removed; persisted or not)
- Identity columns -> `Identity` + `IdentityGeneration = ALWAYS`
## Supported Constraints
- Primary Keys
+37 -2
View File
@@ -104,8 +104,12 @@ func (r *Reader) queryColumns(schemaName string) (map[string]map[string]*models.
c.numeric_precision,
c.numeric_scale,
ISNULL(ep.value, '') as description,
COLUMNPROPERTY(OBJECT_ID(QUOTENAME(c.table_schema) + '.' + QUOTENAME(c.table_name)), c.column_name, 'IsIdentity') as is_identity
COLUMNPROPERTY(OBJECT_ID(QUOTENAME(c.table_schema) + '.' + QUOTENAME(c.table_name)), c.column_name, 'IsIdentity') as is_identity,
cc.definition as computed_definition
FROM information_schema.columns c
LEFT JOIN sys.computed_columns cc
ON cc.object_id = OBJECT_ID(QUOTENAME(c.table_schema) + '.' + QUOTENAME(c.table_name))
AND cc.name = c.column_name
LEFT JOIN sys.extended_properties ep
ON ep.major_id = OBJECT_ID(QUOTENAME(c.table_schema) + '.' + QUOTENAME(c.table_name))
AND ep.minor_id = COLUMNPROPERTY(OBJECT_ID(QUOTENAME(c.table_schema) + '.' + QUOTENAME(c.table_name)), c.column_name, 'ColumnId')
@@ -127,8 +131,9 @@ func (r *Reader) queryColumns(schemaName string) (map[string]map[string]*models.
var schema, tableName, columnName, isNullable, dataType, description string
var ordinalPosition int
var columnDefault, charMaxLength, numPrecision, numScale, isIdentity *int
var computedDefinition *string
if err := rows.Scan(&schema, &tableName, &columnName, &ordinalPosition, &columnDefault, &isNullable, &dataType, &charMaxLength, &numPrecision, &numScale, &description, &isIdentity); err != nil {
if err := rows.Scan(&schema, &tableName, &columnName, &ordinalPosition, &columnDefault, &isNullable, &dataType, &charMaxLength, &numPrecision, &numScale, &description, &isIdentity, &computedDefinition); err != nil {
return nil, err
}
@@ -144,6 +149,14 @@ func (r *Reader) queryColumns(schemaName string) (map[string]map[string]*models.
// Check if this is an identity column (auto-increment)
if isIdentity != nil && *isIdentity == 1 {
column.AutoIncrement = true
column.Identity = true
column.IdentityGeneration = "ALWAYS"
}
// Computed columns report their expression wrapped in an extra pair of parentheses
if computedDefinition != nil && strings.TrimSpace(*computedDefinition) != "" {
column.Generated = true
column.GenerationExpression = trimOuterParens(*computedDefinition)
}
if charMaxLength != nil && *charMaxLength > 0 {
@@ -414,3 +427,25 @@ func (r *Reader) queryIndexes(schemaName string) (map[string][]*models.Index, er
return indexes, rows.Err()
}
// trimOuterParens removes one pair of parentheses wrapping the whole expression, as
// SQL Server stores computed column definitions.
func trimOuterParens(expr string) string {
expr = strings.TrimSpace(expr)
if len(expr) < 2 || expr[0] != '(' || expr[len(expr)-1] != ')' {
return expr
}
depth := 0
for i := 0; i < len(expr); i++ {
switch expr[i] {
case '(':
depth++
case ')':
depth--
if depth == 0 && i != len(expr)-1 {
return expr
}
}
}
return strings.TrimSpace(expr[1 : len(expr)-1])
}
+15
View File
@@ -85,3 +85,18 @@ func TestConvertMSSQLToCanonical(t *testing.T) {
})
}
}
func TestTrimOuterParens(t *testing.T) {
tests := map[string]string{
"(([a])+([b]))": "([a])+([b])",
"([a]+[b])": "[a]+[b]",
"([a])+([b])": "([a])+([b])",
"[a]+[b]": "[a]+[b]",
" (upper([a])) ": "upper([a])",
}
for in, want := range tests {
if got := trimOuterParens(in); got != want {
t.Errorf("trimOuterParens(%q) = %q, want %q", in, got, want)
}
}
}
+245
View File
@@ -0,0 +1,245 @@
package mysql
import (
"context"
"database/sql"
"fmt"
"strings"
_ "github.com/go-sql-driver/mysql"
"git.warky.dev/wdevs/relspecgo/pkg/mariadb"
"git.warky.dev/wdevs/relspecgo/pkg/models"
"git.warky.dev/wdevs/relspecgo/pkg/readers"
)
type Reader struct {
options *readers.ReaderOptions
db *sql.DB
ctx context.Context
}
func NewReader(options *readers.ReaderOptions) *Reader {
return &Reader{options: options, ctx: context.Background()}
}
func (r *Reader) ReadDatabase() (*models.Database, error) {
if r.options == nil || r.options.ConnectionString == "" {
return nil, fmt.Errorf("connection string is required")
}
if err := r.connect(); err != nil {
return nil, fmt.Errorf("failed to connect: %w", err)
}
defer r.close()
var name, version string
if err := r.db.QueryRowContext(r.ctx, "SELECT DATABASE()").Scan(&name); err != nil {
return nil, fmt.Errorf("failed to get database name: %w", err)
}
_ = r.db.QueryRowContext(r.ctx, "SELECT VERSION()").Scan(&version)
db := models.InitDatabase(name)
db.DatabaseType, db.SourceFormat, db.DatabaseVersion = models.MySQLDatabaseType, "mysql", version
schemas, err := r.querySchemas(name)
if err != nil {
return nil, fmt.Errorf("failed to query schemas: %w", err)
}
for _, schema := range schemas {
tables, err := r.queryTables(schema.Name)
if err != nil {
return nil, err
}
schema.Tables = tables
for _, table := range tables {
table.Columns, err = r.queryColumns(schema.Name, table.Name)
if err != nil {
return nil, err
}
table.Constraints, err = r.queryConstraints(schema.Name, table.Name)
if err != nil {
return nil, err
}
table.Indexes, err = r.queryIndexes(schema.Name, table.Name)
if err != nil {
return nil, err
}
table.RefSchema = schema
for _, c := range table.Constraints {
if c.Type == models.ForeignKeyConstraint {
r.deriveRelationship(table, c)
}
}
}
schema.RefDatabase = db
db.Schemas = append(db.Schemas, schema)
}
return db, nil
}
func (r *Reader) ReadSchema() (*models.Schema, error) {
db, err := r.ReadDatabase()
if err != nil {
return nil, err
}
if len(db.Schemas) == 0 {
return nil, fmt.Errorf("no schemas found in database")
}
return db.Schemas[0], nil
}
func (r *Reader) ReadTable() (*models.Table, error) {
s, err := r.ReadSchema()
if err != nil {
return nil, err
}
if len(s.Tables) == 0 {
return nil, fmt.Errorf("no tables found in schema")
}
return s.Tables[0], nil
}
func (r *Reader) connect() error {
db, err := sql.Open("mysql", r.options.ConnectionString)
if err != nil {
return err
}
if err = db.PingContext(r.ctx); err != nil {
db.Close()
return err
}
r.db = db
return nil
}
func (r *Reader) close() {
if r.db != nil {
_ = r.db.Close()
}
}
func (r *Reader) mapDataType(t string) string { return mariadb.ConvertMariaDBToCanonical(t) }
func (r *Reader) querySchemas(current string) ([]*models.Schema, error) {
rows, err := r.db.QueryContext(r.ctx, "SELECT SCHEMA_NAME FROM information_schema.SCHEMATA WHERE SCHEMA_NAME = ?", current)
if err != nil {
return nil, err
}
defer rows.Close()
var out []*models.Schema
for rows.Next() {
var n string
if err := rows.Scan(&n); err != nil {
return nil, err
}
out = append(out, models.InitSchema(n))
}
return out, rows.Err()
}
func (r *Reader) queryTables(schema string) ([]*models.Table, error) {
rows, err := r.db.QueryContext(r.ctx, "SELECT TABLE_NAME FROM information_schema.TABLES WHERE TABLE_SCHEMA = ? AND TABLE_TYPE = 'BASE TABLE' ORDER BY TABLE_NAME", schema)
if err != nil {
return nil, err
}
defer rows.Close()
var out []*models.Table
for rows.Next() {
var n string
if err := rows.Scan(&n); err != nil {
return nil, err
}
out = append(out, models.InitTable(n, schema))
}
return out, rows.Err()
}
func (r *Reader) queryColumns(schema, table string) (map[string]*models.Column, error) {
rows, err := r.db.QueryContext(r.ctx, `SELECT COLUMN_NAME, COLUMN_TYPE, IS_NULLABLE, COLUMN_DEFAULT, ORDINAL_POSITION, EXTRA, COLUMN_COMMENT FROM information_schema.COLUMNS WHERE TABLE_SCHEMA = ? AND TABLE_NAME = ? ORDER BY ORDINAL_POSITION`, schema, table)
if err != nil {
return nil, err
}
defer rows.Close()
out := map[string]*models.Column{}
for rows.Next() {
var name, typ, nullable, extra, comment string
var def sql.NullString
var pos int
if err := rows.Scan(&name, &typ, &nullable, &def, &pos, &extra, &comment); err != nil {
return nil, err
}
c := models.InitColumn(name, table, schema)
c.Type = r.mapDataType(typ)
c.NotNull = strings.EqualFold(nullable, "NO")
c.Sequence = uint(pos)
c.Comment = comment
if def.Valid {
c.Default = def.String
}
c.AutoIncrement = strings.Contains(strings.ToLower(extra), "auto_increment")
out[name] = c
}
return out, rows.Err()
}
func (r *Reader) queryConstraints(schema, table string) (map[string]*models.Constraint, error) {
rows, err := r.db.QueryContext(r.ctx, `SELECT CONSTRAINT_NAME, CONSTRAINT_TYPE, COLUMN_NAME, REFERENCED_TABLE_SCHEMA, REFERENCED_TABLE_NAME, REFERENCED_COLUMN_NAME, ORDINAL_POSITION FROM information_schema.KEY_COLUMN_USAGE k JOIN information_schema.TABLE_CONSTRAINTS t USING (CONSTRAINT_SCHEMA, TABLE_NAME, CONSTRAINT_NAME) WHERE k.TABLE_SCHEMA=? AND k.TABLE_NAME=? ORDER BY CONSTRAINT_NAME, ORDINAL_POSITION`, schema, table)
if err != nil {
return nil, err
}
defer rows.Close()
out := map[string]*models.Constraint{}
for rows.Next() {
var name, typ, col, rs, rt, rc string
var pos int
if err := rows.Scan(&name, &typ, &col, &rs, &rt, &rc, &pos); err != nil {
return nil, err
}
c := out[name]
if c == nil {
ct := models.UniqueConstraint
if typ == "PRIMARY KEY" {
ct = models.PrimaryKeyConstraint
}
if typ == "FOREIGN KEY" {
ct = models.ForeignKeyConstraint
}
c = models.InitConstraint(name, ct)
c.Schema = schema
c.Table = table
c.ReferencedSchema = rs
c.ReferencedTable = rt
out[name] = c
}
c.Columns = append(c.Columns, col)
if rc != "" {
c.ReferencedColumns = append(c.ReferencedColumns, rc)
}
}
return out, rows.Err()
}
func (r *Reader) queryIndexes(schema, table string) (map[string]*models.Index, error) {
rows, err := r.db.QueryContext(r.ctx, `SELECT INDEX_NAME, NON_UNIQUE, COLUMN_NAME, SEQ_IN_INDEX, INDEX_TYPE FROM information_schema.STATISTICS WHERE TABLE_SCHEMA=? AND TABLE_NAME=? AND INDEX_NAME <> 'PRIMARY' ORDER BY INDEX_NAME, SEQ_IN_INDEX`, schema, table)
if err != nil {
return nil, err
}
defer rows.Close()
out := map[string]*models.Index{}
for rows.Next() {
var name, col, typ string
var non, seq int
if err := rows.Scan(&name, &non, &col, &seq, &typ); err != nil {
return nil, err
}
i := out[name]
if i == nil {
i = models.InitIndex(name, table, schema)
i.Unique = non == 0
i.Type = strings.ToLower(typ)
out[name] = i
}
i.Columns = append(i.Columns, col)
}
return out, rows.Err()
}
func (r *Reader) deriveRelationship(t *models.Table, c *models.Constraint) {
n := fmt.Sprintf("%s_to_%s", t.Name, c.ReferencedTable)
rel := models.InitRelationship(n, models.OneToMany)
rel.FromTable = t.Name
rel.FromSchema = t.Schema
rel.FromColumns = append([]string(nil), c.Columns...)
rel.ToTable = c.ReferencedTable
rel.ToSchema = c.ReferencedSchema
rel.ToColumns = append([]string(nil), c.ReferencedColumns...)
rel.ForeignKey = c.Name
t.Relationships[n] = rel
}
+21
View File
@@ -0,0 +1,21 @@
package mysql
import (
"git.warky.dev/wdevs/relspecgo/pkg/readers"
"testing"
)
func TestReaderMapDataType(t *testing.T) {
r := NewReader(&readers.ReaderOptions{})
for _, tc := range []struct{ input, want string }{{"varchar(64)", "string"}, {"bigint unsigned", "int64"}, {"datetime", "timestamp"}, {"json", "json"}} {
if got := r.mapDataType(tc.input); got != tc.want {
t.Errorf("mapDataType(%q) = %q, want %q", tc.input, got, tc.want)
}
}
}
func TestReaderRequiresConnectionString(t *testing.T) {
if _, err := NewReader(&readers.ReaderOptions{}).ReadDatabase(); err == nil {
t.Fatal("expected missing connection string error")
}
}
+72 -27
View File
@@ -243,7 +243,11 @@ func (r *Reader) queryColumns(schemaName string) (map[string]map[string]*models.
c.numeric_scale,
c.udt_name,
pg_catalog.format_type(a.atttypid, a.atttypmod) as formatted_data_type,
col_description((c.table_schema||'.'||c.table_name)::regclass, c.ordinal_position) as description
col_description((c.table_schema||'.'||c.table_name)::regclass, c.ordinal_position) as description,
c.is_generated,
c.generation_expression,
c.is_identity,
c.identity_generation
FROM information_schema.columns c
JOIN pg_catalog.pg_namespace n
ON n.nspname = c.table_schema
@@ -268,17 +272,36 @@ func (r *Reader) queryColumns(schemaName string) (map[string]map[string]*models.
columnsMap := make(map[string]map[string]*models.Column)
for rows.Next() {
var schema, tableName, columnName, isNullable, dataType, udtName, formattedDataType string
var schema, tableName, columnName, isNullable, dataType, udtName, formattedDataType, isGenerated, isIdentity string
var ordinalPosition int
var columnDefault, description *string
var columnDefault, description, generationExpression, identityGeneration *string
var charMaxLength, numPrecision, numScale *int
if err := rows.Scan(&schema, &tableName, &columnName, &ordinalPosition, &columnDefault, &isNullable, &dataType, &charMaxLength, &numPrecision, &numScale, &udtName, &formattedDataType, &description); err != nil {
if err := rows.Scan(&schema, &tableName, &columnName, &ordinalPosition, &columnDefault, &isNullable, &dataType, &charMaxLength, &numPrecision, &numScale, &udtName, &formattedDataType, &description, &isGenerated, &generationExpression, &isIdentity, &identityGeneration); err != nil {
return nil, err
}
column := models.InitColumn(columnName, tableName, schema)
// GENERATED ALWAYS ... STORED columns are computed by Postgres and cannot
// have their default altered/dropped like a regular column.
column.Generated = isGenerated == "ALWAYS"
if generationExpression != nil {
column.GenerationExpression = *generationExpression
}
// GENERATED { ALWAYS | BY DEFAULT } AS IDENTITY columns are driven by an
// internal sequence rather than a literal default (unlike serial columns,
// they carry no pg_attrdef row at all), so they need the same DB-side
// handling as generated columns even though the underlying mechanism differs.
column.Identity = isIdentity == "YES"
if identityGeneration != nil {
column.IdentityGeneration = strings.ToUpper(strings.TrimSpace(*identityGeneration))
}
if column.Identity {
column.AutoIncrement = true
}
// Check if this is a serial type (has nextval default)
hasNextval := false
if columnDefault != nil {
@@ -301,6 +324,24 @@ func (r *Reader) queryColumns(schemaName string) (map[string]map[string]*models.
column.Description = *description
}
if column.Generated {
note := "GENERATED ALWAYS AS (" + column.GenerationExpression + ") STORED"
if column.Description != "" {
column.Description = column.Description + " " + note
} else {
column.Description = note
}
}
if column.Identity {
note := "GENERATED " + column.IdentityGeneration + " AS IDENTITY"
if column.Description != "" {
column.Description = column.Description + " " + note
} else {
column.Description = note
}
}
if charMaxLength != nil {
column.Length = *charMaxLength
}
@@ -381,31 +422,35 @@ func (r *Reader) queryPrimaryKeys(schemaName string) (map[string]*models.Constra
// queryForeignKeys retrieves all foreign key constraints for a schema
// Returns map[schema.table][]*Constraint
func (r *Reader) queryForeignKeys(schemaName string) (map[string][]*models.Constraint, error) {
// Columns are paired by position from pg_constraint so composite keys are not
// cross-joined (information_schema.constraint_column_usage has no ordering).
actionCase := func(col string) string {
return `CASE ` + col + ` WHEN 'a' THEN 'NO ACTION' WHEN 'r' THEN 'RESTRICT' WHEN 'c' THEN 'CASCADE' WHEN 'n' THEN 'SET NULL' WHEN 'd' THEN 'SET DEFAULT' END`
}
query := `
SELECT
tc.table_schema,
tc.table_name,
tc.constraint_name,
kcu.table_schema as foreign_table_schema,
kcu.table_name as foreign_table_name,
kcu.column_name as foreign_column,
ccu.table_schema as referenced_table_schema,
ccu.table_name as referenced_table_name,
ccu.column_name as referenced_column,
rc.update_rule,
rc.delete_rule
FROM information_schema.table_constraints tc
JOIN information_schema.key_column_usage kcu
ON tc.constraint_name = kcu.constraint_name
AND tc.table_schema = kcu.table_schema
JOIN information_schema.constraint_column_usage ccu
ON ccu.constraint_name = tc.constraint_name
JOIN information_schema.referential_constraints rc
ON rc.constraint_name = tc.constraint_name
AND rc.constraint_schema = tc.table_schema
WHERE tc.constraint_type = 'FOREIGN KEY'
AND tc.table_schema = $1
ORDER BY tc.table_schema, tc.table_name, tc.constraint_name, kcu.ordinal_position
ns.nspname AS table_schema,
cl.relname AS table_name,
con.conname AS constraint_name,
ns.nspname AS foreign_table_schema,
cl.relname AS foreign_table_name,
att.attname AS foreign_column,
fns.nspname AS referenced_table_schema,
fcl.relname AS referenced_table_name,
fatt.attname AS referenced_column,
` + actionCase("con.confupdtype") + ` AS update_rule,
` + actionCase("con.confdeltype") + ` AS delete_rule
FROM pg_catalog.pg_constraint con
JOIN pg_catalog.pg_class cl ON cl.oid = con.conrelid
JOIN pg_catalog.pg_namespace ns ON ns.oid = cl.relnamespace
JOIN pg_catalog.pg_class fcl ON fcl.oid = con.confrelid
JOIN pg_catalog.pg_namespace fns ON fns.oid = fcl.relnamespace
CROSS JOIN LATERAL unnest(con.conkey, con.confkey) WITH ORDINALITY AS k(attnum, fattnum, ord)
JOIN pg_catalog.pg_attribute att ON att.attrelid = con.conrelid AND att.attnum = k.attnum
JOIN pg_catalog.pg_attribute fatt ON fatt.attrelid = con.confrelid AND fatt.attnum = k.fattnum
WHERE con.contype = 'f'
AND ns.nspname = $1
ORDER BY ns.nspname, cl.relname, con.conname, k.ord
`
rows, err := r.conn.Query(r.ctx, query, schemaName)
+57 -2
View File
@@ -34,11 +34,14 @@ func (r *Reader) ReadDatabase() (*models.Database, error) {
return nil, fmt.Errorf("connection string is required")
}
// Connect to the database
// Connect to the database. This can take noticeable time across a slow network,
// so report it before the driver starts the connection attempt.
r.progress("Connecting to PostgreSQL...")
if err := r.connect(); err != nil {
return nil, fmt.Errorf("failed to connect: %w", err)
}
defer r.close()
r.progress("Connected. Reading database metadata...")
// Get database name from connection
var dbName string
@@ -60,34 +63,42 @@ func (r *Reader) ReadDatabase() (*models.Database, error) {
}
// Query all schemas
r.progress("Discovering schemas...")
schemas, err := r.querySchemas()
if err != nil {
return nil, fmt.Errorf("failed to query schemas: %w", err)
}
// Process each schema
for _, schema := range schemas {
for schemaIndex, schema := range schemas {
r.progress(fmt.Sprintf("Reading schema %q (%d/%d): tables...", schema.Name, schemaIndex+1, len(schemas)))
// Query tables for this schema
tables, err := r.queryTables(schema.Name)
if err != nil {
return nil, fmt.Errorf("failed to query tables for schema %s: %w", schema.Name, err)
}
schema.Tables = tables
r.progress(fmt.Sprintf("Reading schema %q: found %d table(s).", schema.Name, len(tables)))
r.progress(fmt.Sprintf("Reading schema %q: views...", schema.Name))
// Query views for this schema
views, err := r.queryViews(schema.Name)
if err != nil {
return nil, fmt.Errorf("failed to query views for schema %s: %w", schema.Name, err)
}
schema.Views = views
r.progress(fmt.Sprintf("Reading schema %q: found %d view(s).", schema.Name, len(views)))
r.progress(fmt.Sprintf("Reading schema %q: sequences...", schema.Name))
// Query sequences for this schema
sequences, err := r.querySequences(schema.Name)
if err != nil {
return nil, fmt.Errorf("failed to query sequences for schema %s: %w", schema.Name, err)
}
schema.Sequences = sequences
r.progress(fmt.Sprintf("Reading schema %q: found %d sequence(s).", schema.Name, len(sequences)))
r.progress(fmt.Sprintf("Reading schema %q: extensions...", schema.Name))
// Query extensions installed into this schema
extensions, err := r.queryExtensions(schema.Name)
if err != nil {
@@ -99,12 +110,15 @@ func (r *Reader) ReadDatabase() (*models.Database, error) {
}
schema.Metadata["extensions"] = extensions
}
r.progress(fmt.Sprintf("Reading schema %q: found %d extension(s).", schema.Name, len(extensions)))
r.progress(fmt.Sprintf("Reading schema %q: columns...", schema.Name))
// Query columns for tables and views
columnsMap, err := r.queryColumns(schema.Name)
if err != nil {
return nil, fmt.Errorf("failed to query columns for schema %s: %w", schema.Name, err)
}
r.progress(fmt.Sprintf("Reading schema %q: found %d column(s).", schema.Name, countColumns(columnsMap)))
// Populate table columns
for _, table := range schema.Tables {
@@ -122,11 +136,13 @@ func (r *Reader) ReadDatabase() (*models.Database, error) {
}
}
r.progress(fmt.Sprintf("Reading schema %q: primary keys...", schema.Name))
// Query primary keys
primaryKeys, err := r.queryPrimaryKeys(schema.Name)
if err != nil {
return nil, fmt.Errorf("failed to query primary keys for schema %s: %w", schema.Name, err)
}
r.progress(fmt.Sprintf("Reading schema %q: found %d primary key(s).", schema.Name, len(primaryKeys)))
// Apply primary keys to tables
for _, table := range schema.Tables {
@@ -143,11 +159,13 @@ func (r *Reader) ReadDatabase() (*models.Database, error) {
}
}
r.progress(fmt.Sprintf("Reading schema %q: foreign keys...", schema.Name))
// Query foreign keys
foreignKeys, err := r.queryForeignKeys(schema.Name)
if err != nil {
return nil, fmt.Errorf("failed to query foreign keys for schema %s: %w", schema.Name, err)
}
r.progress(fmt.Sprintf("Reading schema %q: found %d foreign key(s).", schema.Name, countConstraints(foreignKeys)))
// Apply foreign keys to tables
for _, table := range schema.Tables {
@@ -161,11 +179,13 @@ func (r *Reader) ReadDatabase() (*models.Database, error) {
}
}
r.progress(fmt.Sprintf("Reading schema %q: unique constraints...", schema.Name))
// Query unique constraints
uniqueConstraints, err := r.queryUniqueConstraints(schema.Name)
if err != nil {
return nil, fmt.Errorf("failed to query unique constraints for schema %s: %w", schema.Name, err)
}
r.progress(fmt.Sprintf("Reading schema %q: found %d unique constraint(s).", schema.Name, countConstraints(uniqueConstraints)))
// Apply unique constraints to tables
for _, table := range schema.Tables {
@@ -177,11 +197,13 @@ func (r *Reader) ReadDatabase() (*models.Database, error) {
}
}
r.progress(fmt.Sprintf("Reading schema %q: check constraints...", schema.Name))
// Query check constraints
checkConstraints, err := r.queryCheckConstraints(schema.Name)
if err != nil {
return nil, fmt.Errorf("failed to query check constraints for schema %s: %w", schema.Name, err)
}
r.progress(fmt.Sprintf("Reading schema %q: found %d check constraint(s).", schema.Name, countConstraints(checkConstraints)))
// Apply check constraints to tables
for _, table := range schema.Tables {
@@ -193,11 +215,13 @@ func (r *Reader) ReadDatabase() (*models.Database, error) {
}
}
r.progress(fmt.Sprintf("Reading schema %q: indexes...", schema.Name))
// Query indexes
indexes, err := r.queryIndexes(schema.Name)
if err != nil {
return nil, fmt.Errorf("failed to query indexes for schema %s: %w", schema.Name, err)
}
r.progress(fmt.Sprintf("Reading schema %q: found %d index(es).", schema.Name, countIndexes(indexes)))
// Apply indexes to tables
for _, table := range schema.Tables {
@@ -226,10 +250,41 @@ func (r *Reader) ReadDatabase() (*models.Database, error) {
// Add schema to database
db.Schemas = append(db.Schemas, schema)
}
r.progress("PostgreSQL schema read complete.")
return db, nil
}
func (r *Reader) progress(message string) {
if r.options.Progress != nil {
r.options.Progress(message)
}
}
func countColumns(columns map[string]map[string]*models.Column) int {
total := 0
for _, tableColumns := range columns {
total += len(tableColumns)
}
return total
}
func countConstraints(constraints map[string][]*models.Constraint) int {
total := 0
for _, tableConstraints := range constraints {
total += len(tableConstraints)
}
return total
}
func countIndexes(indexes map[string][]*models.Index) int {
total := 0
for _, tableIndexes := range indexes {
total += len(tableIndexes)
}
return total
}
// ReadSchema reads a single schema (returns the first schema from the database)
func (r *Reader) ReadSchema() (*models.Schema, error) {
db, err := r.ReadDatabase()
+61
View File
@@ -1,10 +1,14 @@
package pgsql
import (
"context"
"os"
"reflect"
"strings"
"testing"
"github.com/jackc/pgx/v5"
"git.warky.dev/wdevs/relspecgo/pkg/models"
"git.warky.dev/wdevs/relspecgo/pkg/readers"
)
@@ -499,3 +503,60 @@ func TestMapDataType_ExtensionTypesPreserveModifiers(t *testing.T) {
})
}
}
func TestReader_CompositeForeignKeyColumnsArePairedOnce(t *testing.T) {
connStr := getTestConnectionString(t)
ctx := context.Background()
conn, err := pgx.Connect(ctx, connStr)
if err != nil {
t.Fatalf("connect: %v", err)
}
defer conn.Close(ctx)
const schema = "relspec_fk_test"
setup := []string{
"DROP SCHEMA IF EXISTS " + schema + " CASCADE",
"CREATE SCHEMA " + schema,
"CREATE TABLE " + schema + ".parent (a int, b int, PRIMARY KEY (a, b))",
"CREATE TABLE " + schema + ".child (x int, y int, CONSTRAINT fk_child_parent FOREIGN KEY (x, y) REFERENCES " + schema + ".parent (a, b) ON DELETE CASCADE)",
}
for _, stmt := range setup {
if _, err := conn.Exec(ctx, stmt); err != nil {
t.Fatalf("setup %q: %v", stmt, err)
}
}
defer conn.Exec(ctx, "DROP SCHEMA IF EXISTS "+schema+" CASCADE")
reader := NewReader(&readers.ReaderOptions{ConnectionString: connStr})
db, err := reader.ReadDatabase()
if err != nil {
t.Fatalf("ReadDatabase: %v", err)
}
for _, s := range db.Schemas {
if s.Name != schema {
continue
}
for _, tbl := range s.Tables {
if tbl.Name != "child" {
continue
}
fk := tbl.Constraints["fk_child_parent"]
if fk == nil {
t.Fatal("foreign key fk_child_parent not read")
}
if got := strings.Join(fk.Columns, ","); got != "x,y" {
t.Errorf("columns = %q, want x,y", got)
}
if got := strings.Join(fk.ReferencedColumns, ","); got != "a,b" {
t.Errorf("referenced columns = %q, want a,b", got)
}
if fk.OnDelete != "CASCADE" || fk.OnUpdate != "NO ACTION" {
t.Errorf("rules = %s/%s, want CASCADE/NO ACTION", fk.OnDelete, fk.OnUpdate)
}
return
}
}
t.Fatal("test schema/table not found in read result")
}
+4
View File
@@ -32,6 +32,10 @@ type ReaderOptions struct {
// fail on an unknown namespace or key instead of preserving them silently.
StrictDirectives bool
// Progress receives human-readable status updates while a reader is working.
// It is optional so library users can opt in without coupling readers to a UI.
Progress func(string)
// Additional options can be added here as needed
Metadata map[string]interface{}
}
+1
View File
@@ -40,6 +40,7 @@ options := &readers.ReaderOptions{
- Uses pure Go driver (modernc.org/sqlite) - no CGo required
- Supports both file path and connection string
- Auto-increment detection for INTEGER PRIMARY KEY columns
- Generated columns (virtual and stored) are read via `PRAGMA table_xinfo`; the expression is parsed from the `CREATE TABLE` SQL
- Foreign keys require `PRAGMA foreign_keys = ON` to be set
## Example Schema
+116
View File
@@ -0,0 +1,116 @@
package sqlite
import (
"regexp"
"strings"
)
var generatedAsRegex = regexp.MustCompile(`(?is)\bAS\s*\(`)
// parseGeneratedExpression extracts the expression of a generated column from a CREATE
// TABLE statement, e.g. `full TEXT GENERATED ALWAYS AS (a || b) STORED` yields `a || b`.
// It returns "" when the column or its expression cannot be found.
func parseGeneratedExpression(createSQL, columnName string) string {
open := strings.Index(createSQL, "(")
if open < 0 {
return ""
}
for _, def := range splitTopLevel(createSQL[open+1:]) {
if !strings.EqualFold(firstIdentifier(def), columnName) {
continue
}
loc := generatedAsRegex.FindStringIndex(def)
if loc == nil {
return ""
}
return balancedParens(def[loc[1]-1:])
}
return ""
}
// splitTopLevel splits a table body on commas that are outside parentheses and quotes,
// stopping at the parenthesis that closes the body.
func splitTopLevel(body string) []string {
var parts []string
depth := 0
var quote byte
start := 0
for i := 0; i < len(body); i++ {
ch := body[i]
if quote != 0 {
if ch == quote {
quote = 0
}
continue
}
switch ch {
case '\'', '"', '`':
quote = ch
case '[':
quote = ']'
case '(':
depth++
case ')':
if depth == 0 {
return append(parts, body[start:i])
}
depth--
case ',':
if depth == 0 {
parts = append(parts, body[start:i])
start = i + 1
}
}
}
return append(parts, body[start:])
}
// firstIdentifier returns the first (possibly quoted) identifier of a column definition.
func firstIdentifier(def string) string {
def = strings.TrimSpace(def)
if def == "" {
return ""
}
switch def[0] {
case '"', '\'', '`':
if end := strings.IndexByte(def[1:], def[0]); end >= 0 {
return def[1 : 1+end]
}
case '[':
if end := strings.IndexByte(def, ']'); end >= 0 {
return def[1:end]
}
}
if end := strings.IndexAny(def, " \t\r\n"); end >= 0 {
return def[:end]
}
return def
}
// balancedParens returns the text inside the parenthesis group that starts at s[0].
func balancedParens(s string) string {
depth := 0
var quote byte
for i := 0; i < len(s); i++ {
ch := s[i]
if quote != 0 {
if ch == quote {
quote = 0
}
continue
}
switch ch {
case '\'', '"':
quote = ch
case '(':
depth++
case ')':
depth--
if depth == 0 {
return strings.TrimSpace(s[1:i])
}
}
}
return ""
}
+64
View File
@@ -0,0 +1,64 @@
package sqlite
import (
"database/sql"
"path/filepath"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"git.warky.dev/wdevs/relspecgo/pkg/readers"
)
func TestParseGeneratedExpression(t *testing.T) {
createSQL := `CREATE TABLE people (
id INTEGER PRIMARY KEY,
"first" TEXT,
last TEXT,
full_name TEXT GENERATED ALWAYS AS (coalesce("first", '') || ' ' || last) STORED,
initials TEXT AS (substr("first", 1, 1) || substr(last, 1, 1)),
plain TEXT NOT NULL
)`
tests := []struct {
column string
want string
}{
{"full_name", `coalesce("first", '') || ' ' || last`},
{"initials", `substr("first", 1, 1) || substr(last, 1, 1)`},
{"plain", ""},
{"missing", ""},
}
for _, tt := range tests {
t.Run(tt.column, func(t *testing.T) {
assert.Equal(t, tt.want, parseGeneratedExpression(createSQL, tt.column))
})
}
}
func TestReader_GeneratedColumns(t *testing.T) {
dbPath := filepath.Join(t.TempDir(), "gen.db")
db, err := sql.Open("sqlite", dbPath)
require.NoError(t, err)
_, err = db.Exec(`CREATE TABLE people (
id INTEGER PRIMARY KEY,
first TEXT,
last TEXT,
full_name TEXT GENERATED ALWAYS AS (first || ' ' || last) STORED,
initials TEXT AS (substr(first, 1, 1) || substr(last, 1, 1))
)`)
require.NoError(t, err)
require.NoError(t, db.Close())
got, err := NewReader(&readers.ReaderOptions{FilePath: dbPath}).ReadDatabase()
require.NoError(t, err)
cols := got.Schemas[0].Tables[0].Columns
require.Contains(t, cols, "full_name", "generated columns must be read")
assert.True(t, cols["full_name"].Generated)
assert.Equal(t, "first || ' ' || last", cols["full_name"].GenerationExpression)
assert.True(t, cols["initials"].Generated)
assert.Equal(t, "substr(first, 1, 1) || substr(last, 1, 1)", cols["initials"].GenerationExpression)
assert.False(t, cols["first"].Generated)
}
+29 -4
View File
@@ -75,7 +75,8 @@ func (r *Reader) queryViews() ([]*models.View, error) {
// queryColumns retrieves all columns for a given table or view
func (r *Reader) queryColumns(tableName string) (map[string]*models.Column, error) {
query := fmt.Sprintf("PRAGMA table_info(%s)", tableName)
// table_xinfo, unlike table_info, also lists generated columns (hidden = 2 virtual, 3 stored)
query := fmt.Sprintf("PRAGMA table_xinfo(%s)", tableName)
rows, err := r.db.QueryContext(r.ctx, query)
if err != nil {
@@ -84,24 +85,38 @@ func (r *Reader) queryColumns(tableName string) (map[string]*models.Column, erro
defer rows.Close()
columns := make(map[string]*models.Column)
var tableSQL string
tableSQLLoaded := false
for rows.Next() {
var cid int
var name, dataType string
var notNull, pk int
var notNull, pk, hidden int
var defaultValue *string
if err := rows.Scan(&cid, &name, &dataType, &notNull, &defaultValue, &pk); err != nil {
if err := rows.Scan(&cid, &name, &dataType, &notNull, &defaultValue, &pk, &hidden); err != nil {
return nil, err
}
// Hidden virtual-table columns (hidden = 1) are not part of the schema
if hidden == 1 {
continue
}
column := models.InitColumn(name, tableName, "main")
column.Type = r.mapDataType(strings.ToUpper(dataType))
column.NotNull = (notNull == 1)
column.IsPrimaryKey = (pk > 0)
column.Sequence = uint(cid + 1)
if defaultValue != nil {
if hidden == 2 || hidden == 3 {
column.Generated = true
if !tableSQLLoaded {
tableSQL = r.tableSQL(tableName)
tableSQLLoaded = true
}
column.GenerationExpression = parseGeneratedExpression(tableSQL, name)
} else if defaultValue != nil {
column.Default = *defaultValue
}
@@ -116,6 +131,16 @@ func (r *Reader) queryColumns(tableName string) (map[string]*models.Column, erro
return columns, rows.Err()
}
// tableSQL returns the CREATE TABLE statement of a table, or "" when it cannot be read.
func (r *Reader) tableSQL(tableName string) string {
var sql string
err := r.db.QueryRowContext(r.ctx, `SELECT sql FROM sqlite_master WHERE type = 'table' AND name = ?`, tableName).Scan(&sql)
if err != nil {
return ""
}
return sql
}
// isAutoIncrement checks if a column is autoincrement
func (r *Reader) isAutoIncrement(tableName, columnName string) bool {
// Check sqlite_sequence table or parse CREATE TABLE statement
+2
View File
@@ -114,6 +114,8 @@ export class Post {
- `@JoinColumn()` - Foreign key column
- `@Index()` - Index definition
- `@Unique()` - Unique constraint
- `asExpression` / `generatedType` - Generated column (`Generated` + `GenerationExpression`)
- `@PrimaryGeneratedColumn('identity')`, `@Generated('identity')`, `generatedIdentity` - Identity column
## Notes
+94 -4
View File
@@ -128,7 +128,6 @@ func (r *Reader) extractEntities(content string) []entityInfo {
scanner := bufio.NewScanner(strings.NewReader(content))
entityRegex := regexp.MustCompile(`^export\s+class\s+(\w+)`)
decoratorRegex := regexp.MustCompile(`^\s*@(\w+)(\([^)]*\))?`)
fieldRegex := regexp.MustCompile(`^\s*(\w+):\s*([^;]+);`)
var currentEntity *entityInfo
@@ -145,8 +144,7 @@ func (r *Reader) extractEntities(content string) []entityInfo {
}
// Check for decorator
if matches := decoratorRegex.FindStringSubmatch(trimmed); matches != nil {
decorator := matches[0]
if decorator, ok := matchDecorator(trimmed); ok {
pendingDecorators = append(pendingDecorators, decorator)
continue
}
@@ -488,7 +486,11 @@ func (r *Reader) parseColumnDecorator(decorator string, column *models.Column, t
column.IsPrimaryKey = true
column.NotNull = true
if strings.Contains(decorator, "'uuid'") {
if strings.Contains(decorator, "'identity'") {
column.AutoIncrement = true
column.Identity = true
column.IdentityGeneration = parseGeneratedIdentity(decorator)
} else if strings.Contains(decorator, "'uuid'") {
column.Type = "uuid"
column.Default = "gen_random_uuid()"
} else if strings.Contains(decorator, "'increment'") || strings.Contains(decorator, "()") {
@@ -497,6 +499,17 @@ func (r *Reader) parseColumnDecorator(decorator string, column *models.Column, t
return
}
// @Generated('identity') on a non-key column; generatedIdentity is read from @Column
if strings.HasPrefix(decorator, "@Generated") {
if strings.Contains(decorator, "'identity'") {
column.Identity = true
if column.IdentityGeneration == "" {
column.IdentityGeneration = "BY DEFAULT"
}
}
return
}
// @Column
if strings.HasPrefix(decorator, "@Column") {
r.parseColumnOptions(decorator, column, table)
@@ -586,6 +599,14 @@ func (r *Reader) parseColumnOptions(decorator string, column *models.Column, tab
}
}
if matches := asExpressionRegex.FindStringSubmatch(content); matches != nil {
column.Generated = true
column.GenerationExpression = unescapeSingleQuoted(matches[1])
}
if strings.Contains(content, "generatedIdentity") {
column.IdentityGeneration = parseGeneratedIdentity(content)
}
if strings.Contains(content, "nullable: true") || strings.Contains(content, "nullable:true") {
column.NotNull = false
}
@@ -834,3 +855,72 @@ func (r *Reader) getPrimaryKeyColumn(table *models.Table) *models.Column {
return pk
}
var (
asExpressionRegex = regexp.MustCompile(`asExpression:\s*'((?:\\.|[^\\'])*)'`)
generatedIdentityRegexp = regexp.MustCompile(`generatedIdentity:\s*['"](ALWAYS|BY DEFAULT)['"]`)
)
// parseGeneratedIdentity returns the identity generation mode named in a decorator
// ("ALWAYS" or "BY DEFAULT"), defaulting to "BY DEFAULT" as TypeORM does.
func parseGeneratedIdentity(decorator string) string {
if matches := generatedIdentityRegexp.FindStringSubmatch(decorator); matches != nil {
return matches[1]
}
return "BY DEFAULT"
}
// unescapeSingleQuoted reverses escaping applied inside a quoted TypeScript string.
func unescapeSingleQuoted(s string) string {
var sb strings.Builder
for i := 0; i < len(s); i++ {
if s[i] == '\\' && i+1 < len(s) {
i++
}
sb.WriteByte(s[i])
}
return sb.String()
}
var decoratorNameRegex = regexp.MustCompile(`^@\w+`)
// matchDecorator returns the decorator at the start of line, including its argument list.
// Parentheses inside quoted strings (e.g. a generated column expression) do not end it.
func matchDecorator(line string) (string, bool) {
name := decoratorNameRegex.FindString(line)
if name == "" {
return "", false
}
rest := line[len(name):]
if !strings.HasPrefix(rest, "(") {
return name, true
}
depth := 0
var quote byte
for i := 0; i < len(rest); i++ {
ch := rest[i]
if quote != 0 {
switch ch {
case '\\':
i++
case quote:
quote = 0
}
continue
}
switch ch {
case '\'', '"', '`':
quote = ch
case '(':
depth++
case ')':
depth--
if depth == 0 {
return name + rest[:i+1], true
}
}
}
// Unterminated argument list: keep the whole line, as a best effort
return line, true
}
+4
View File
@@ -207,6 +207,10 @@ func (se *SchemaEditor) showDomainEditor(index int, domain *models.Domain) {
se.showDomainList()
})
form.AddButton("Tables", func() {
se.showDomainTables(index)
})
form.AddButton("Delete", func() {
se.showDeleteDomainConfirm(index)
})
+12
View File
@@ -39,6 +39,18 @@ func (se *SchemaEditor) createMainMenu() tview.Primitive {
AddItem("Manage Domains", "View, create, edit, and delete domains", 'd', func() {
se.showDomainList()
}).
AddItem("Manage Indexes", "View, create, edit, and delete table indexes", 'x', func() {
se.showObjectList(se.indexKind())
}).
AddItem("Manage Views", "View, create, edit, and delete views", 'v', func() {
se.showObjectList(se.viewKind())
}).
AddItem("Manage Sequences", "View, create, edit, and delete sequences", 'u', func() {
se.showObjectList(se.sequenceKind())
}).
AddItem("Manage Scripts", "View, create, edit, and delete SQL scripts", 'c', func() {
se.showObjectList(se.scriptKind())
}).
AddItem("Import & Merge", "Import and merge schema from another database", 'i', func() {
se.showImportScreen()
}).
+263
View File
@@ -0,0 +1,263 @@
package ui
import (
"errors"
"fmt"
"strings"
"git.warky.dev/wdevs/relspecgo/pkg/models"
)
// Data operations for indexes, views, sequences, scripts and domain/table assignment.
func (se *SchemaEditor) schemaAt(schemaIndex int) (*models.Schema, error) {
if schemaIndex < 0 || schemaIndex >= len(se.db.Schemas) {
return nil, errors.New("schema not found")
}
return se.db.Schemas[schemaIndex], nil
}
func (se *SchemaEditor) tableAt(schemaIndex, tableIndex int) (*models.Schema, *models.Table, error) {
schema, err := se.schemaAt(schemaIndex)
if err != nil {
return nil, nil, err
}
if tableIndex < 0 || tableIndex >= len(schema.Tables) {
return nil, nil, errors.New("table not found")
}
return schema, schema.Tables[tableIndex], nil
}
// splitList splits a comma separated list, trimming blanks and dropping empty entries.
func splitList(s string) []string {
parts := make([]string, 0)
for _, p := range strings.Split(s, ",") {
if p = strings.TrimSpace(p); p != "" {
parts = append(parts, p)
}
}
return parts
}
// SaveIndex adds an index to a table. When oldName is non-empty the index of that
// name is replaced (and renamed if needed).
func (se *SchemaEditor) SaveIndex(schemaIndex, tableIndex int, oldName string, idx *models.Index) error {
schema, table, err := se.tableAt(schemaIndex, tableIndex)
if err != nil {
return err
}
idx.Name = strings.TrimSpace(idx.Name)
if idx.Name == "" {
return errors.New("index name is required")
}
if len(idx.Columns) == 0 {
return errors.New("index needs at least one column")
}
for _, c := range idx.Columns {
if _, ok := table.Columns[c]; !ok {
return fmt.Errorf("column %q not found in table %s", c, table.Name)
}
}
if _, exists := table.Indexes[idx.Name]; exists && idx.Name != oldName {
return fmt.Errorf("index %q already exists", idx.Name)
}
if table.Indexes == nil {
table.Indexes = make(map[string]*models.Index)
}
if oldName != "" {
delete(table.Indexes, oldName)
}
idx.Table = table.Name
idx.Schema = schema.Name
table.Indexes[idx.Name] = idx
table.UpdateDate()
se.db.UpdateDate()
return nil
}
// DeleteIndex removes an index from a table.
func (se *SchemaEditor) DeleteIndex(schemaIndex, tableIndex int, name string) bool {
_, table, err := se.tableAt(schemaIndex, tableIndex)
if err != nil {
return false
}
if _, ok := table.Indexes[name]; !ok {
return false
}
delete(table.Indexes, name)
table.UpdateDate()
se.db.UpdateDate()
return true
}
// SaveView adds a view to a schema, or replaces the one at position at (use -1 to add).
func (se *SchemaEditor) SaveView(schemaIndex, at int, v *models.View) error {
schema, err := se.schemaAt(schemaIndex)
if err != nil {
return err
}
v.Name = strings.TrimSpace(v.Name)
if v.Name == "" {
return errors.New("view name is required")
}
if strings.TrimSpace(v.Definition) == "" {
return errors.New("view definition is required")
}
for i, o := range schema.Views {
if i != at && o.Name == v.Name {
return fmt.Errorf("view %q already exists", v.Name)
}
}
v.Schema = schema.Name
if at >= 0 && at < len(schema.Views) {
schema.Views[at] = v
} else {
schema.Views = append(schema.Views, v)
}
schema.UpdateDate()
se.db.UpdateDate()
return nil
}
// DeleteView removes the view at position at.
func (se *SchemaEditor) DeleteView(schemaIndex, at int) bool {
schema, err := se.schemaAt(schemaIndex)
if err != nil || at < 0 || at >= len(schema.Views) {
return false
}
schema.Views = append(schema.Views[:at], schema.Views[at+1:]...)
schema.UpdateDate()
se.db.UpdateDate()
return true
}
// SaveSequence adds a sequence to a schema, or replaces the one at position at (use -1 to add).
func (se *SchemaEditor) SaveSequence(schemaIndex, at int, s *models.Sequence) error {
schema, err := se.schemaAt(schemaIndex)
if err != nil {
return err
}
s.Name = strings.TrimSpace(s.Name)
if s.Name == "" {
return errors.New("sequence name is required")
}
if s.IncrementBy == 0 {
return errors.New("increment must not be zero")
}
for i, o := range schema.Sequences {
if i != at && o.Name == s.Name {
return fmt.Errorf("sequence %q already exists", s.Name)
}
}
s.Schema = schema.Name
if at >= 0 && at < len(schema.Sequences) {
schema.Sequences[at] = s
} else {
schema.Sequences = append(schema.Sequences, s)
}
schema.UpdateDate()
se.db.UpdateDate()
return nil
}
// DeleteSequence removes the sequence at position at.
func (se *SchemaEditor) DeleteSequence(schemaIndex, at int) bool {
schema, err := se.schemaAt(schemaIndex)
if err != nil || at < 0 || at >= len(schema.Sequences) {
return false
}
schema.Sequences = append(schema.Sequences[:at], schema.Sequences[at+1:]...)
schema.UpdateDate()
se.db.UpdateDate()
return true
}
// SaveScript adds a script to a schema, or replaces the one at position at (use -1 to add).
func (se *SchemaEditor) SaveScript(schemaIndex, at int, s *models.Script) error {
schema, err := se.schemaAt(schemaIndex)
if err != nil {
return err
}
s.Name = strings.TrimSpace(s.Name)
if s.Name == "" {
return errors.New("script name is required")
}
if strings.TrimSpace(s.SQL) == "" {
return errors.New("script SQL is required")
}
for i, o := range schema.Scripts {
if i != at && o.Name == s.Name {
return fmt.Errorf("script %q already exists", s.Name)
}
}
s.Schema = schema.Name
if at >= 0 && at < len(schema.Scripts) {
schema.Scripts[at] = s
} else {
schema.Scripts = append(schema.Scripts, s)
}
schema.UpdateDate()
se.db.UpdateDate()
return nil
}
// DeleteScript removes the script at position at.
func (se *SchemaEditor) DeleteScript(schemaIndex, at int) bool {
schema, err := se.schemaAt(schemaIndex)
if err != nil || at < 0 || at >= len(schema.Scripts) {
return false
}
schema.Scripts = append(schema.Scripts[:at], schema.Scripts[at+1:]...)
schema.UpdateDate()
se.db.UpdateDate()
return true
}
// AssignTableToDomain adds a reference to schemaName.tableName to the domain at domainIndex.
func (se *SchemaEditor) AssignTableToDomain(domainIndex int, schemaName, tableName string) error {
if domainIndex < 0 || domainIndex >= len(se.db.Domains) {
return errors.New("domain not found")
}
domain := se.db.Domains[domainIndex]
var table *models.Table
for _, s := range se.db.Schemas {
if s.Name != schemaName {
continue
}
for _, t := range s.Tables {
if t.Name == tableName {
table = t
}
}
}
if table == nil {
return fmt.Errorf("table %s.%s not found", schemaName, tableName)
}
for _, dt := range domain.Tables {
if dt.SchemaName == schemaName && dt.TableName == tableName {
return fmt.Errorf("table %s.%s is already in domain %s", schemaName, tableName, domain.Name)
}
}
dt := models.InitDomainTable(tableName, schemaName)
dt.RefTable = table
dt.Sequence = uint(len(domain.Tables))
domain.Tables = append(domain.Tables, dt)
se.db.UpdateDate()
return nil
}
// UnassignTableFromDomain removes the reference to schemaName.tableName from the domain.
func (se *SchemaEditor) UnassignTableFromDomain(domainIndex int, schemaName, tableName string) bool {
if domainIndex < 0 || domainIndex >= len(se.db.Domains) {
return false
}
domain := se.db.Domains[domainIndex]
for i, dt := range domain.Tables {
if dt.SchemaName == schemaName && dt.TableName == tableName {
domain.Tables = append(domain.Tables[:i], domain.Tables[i+1:]...)
se.db.UpdateDate()
return true
}
}
return false
}
+136
View File
@@ -0,0 +1,136 @@
package ui
import (
"testing"
"git.warky.dev/wdevs/relspecgo/pkg/models"
)
func newTestEditor() *SchemaEditor {
db := models.InitDatabase("test")
schema := models.InitSchema("public")
table := models.InitTable("users", "public")
table.Columns["id"] = models.InitColumn("id", "users", "public")
table.Columns["email"] = models.InitColumn("email", "users", "public")
schema.Tables = append(schema.Tables, table)
db.Schemas = append(db.Schemas, schema)
return &SchemaEditor{db: db}
}
func TestSaveIndex(t *testing.T) {
se := newTestEditor()
table := se.db.Schemas[0].Tables[0]
tests := []struct {
name string
old string
idx *models.Index
wantErr bool
}{
{"valid", "", &models.Index{Name: "idx_email", Columns: []string{"email"}, Unique: true}, false},
{"duplicate", "", &models.Index{Name: "idx_email", Columns: []string{"email"}}, true},
{"missing name", "", &models.Index{Columns: []string{"email"}}, true},
{"no columns", "", &models.Index{Name: "idx_none"}, true},
{"unknown column", "", &models.Index{Name: "idx_bad", Columns: []string{"nope"}}, true},
{"rename", "idx_email", &models.Index{Name: "idx_email2", Columns: []string{"email", "id"}}, false},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if err := se.SaveIndex(0, 0, tt.old, tt.idx); (err != nil) != tt.wantErr {
t.Fatalf("err = %v, wantErr %v", err, tt.wantErr)
}
})
}
if _, ok := table.Indexes["idx_email"]; ok {
t.Error("renamed index should be gone under old name")
}
if idx := table.Indexes["idx_email2"]; idx == nil || idx.Table != "users" || idx.Schema != "public" {
t.Errorf("unexpected renamed index: %+v", idx)
}
if !se.DeleteIndex(0, 0, "idx_email2") || se.DeleteIndex(0, 0, "idx_email2") {
t.Error("delete should succeed once")
}
}
func TestSaveViewSequenceScript(t *testing.T) {
se := newTestEditor()
schema := se.db.Schemas[0]
if err := se.SaveView(0, -1, &models.View{Name: "v", Definition: "select 1"}); err != nil {
t.Fatal(err)
}
if err := se.SaveView(0, -1, &models.View{Name: "v", Definition: "select 2"}); err == nil {
t.Error("duplicate view accepted")
}
if err := se.SaveView(0, 0, &models.View{Name: "v", Definition: "select 3"}); err != nil {
t.Errorf("editing in place should not conflict: %v", err)
}
if err := se.SaveView(0, -1, &models.View{Name: "w"}); err == nil {
t.Error("view without definition accepted")
}
if len(schema.Views) != 1 || schema.Views[0].Definition != "select 3" || schema.Views[0].Schema != "public" {
t.Errorf("unexpected views: %+v", schema.Views)
}
if !se.DeleteView(0, 0) || se.DeleteView(0, 0) {
t.Error("view delete mismatch")
}
if err := se.SaveSequence(0, -1, &models.Sequence{Name: "s", IncrementBy: 1, StartValue: 1}); err != nil {
t.Fatal(err)
}
if err := se.SaveSequence(0, -1, &models.Sequence{Name: "z"}); err == nil {
t.Error("zero increment accepted")
}
if !se.DeleteSequence(0, 0) || len(schema.Sequences) != 0 {
t.Error("sequence delete failed")
}
if err := se.SaveScript(0, -1, &models.Script{Name: "init", SQL: "select 1"}); err != nil {
t.Fatal(err)
}
if err := se.SaveScript(0, -1, &models.Script{Name: "empty"}); err == nil {
t.Error("script without SQL accepted")
}
if err := se.SaveScript(5, -1, &models.Script{Name: "x", SQL: "y"}); err == nil {
t.Error("bad schema index accepted")
}
if !se.DeleteScript(0, 0) || len(schema.Scripts) != 0 {
t.Error("script delete failed")
}
}
func TestDomainTableAssignment(t *testing.T) {
se := newTestEditor()
se.createDomainNoUI("core")
if err := se.AssignTableToDomain(0, "public", "users"); err != nil {
t.Fatal(err)
}
if err := se.AssignTableToDomain(0, "public", "users"); err == nil {
t.Error("duplicate assignment accepted")
}
if err := se.AssignTableToDomain(0, "public", "missing"); err == nil {
t.Error("unknown table accepted")
}
if err := se.AssignTableToDomain(3, "public", "users"); err == nil {
t.Error("bad domain index accepted")
}
dt := se.db.Domains[0].Tables[0]
if dt.RefTable != se.db.Schemas[0].Tables[0] {
t.Error("RefTable not linked")
}
if !se.UnassignTableFromDomain(0, "public", "users") || se.UnassignTableFromDomain(0, "public", "users") {
t.Error("unassign mismatch")
}
}
func (se *SchemaEditor) createDomainNoUI(name string) {
se.db.Domains = append(se.db.Domains, models.InitDomain(name))
}
func TestSplitList(t *testing.T) {
got := splitList(" a, b,, c ,")
if len(got) != 3 || got[0] != "a" || got[2] != "c" {
t.Errorf("got %v", got)
}
}
+476
View File
@@ -0,0 +1,476 @@
package ui
import (
"fmt"
"sort"
"strconv"
"strings"
"github.com/gdamore/tcell/v2"
"github.com/rivo/tview"
"git.warky.dev/wdevs/relspecgo/pkg/models"
)
// objectLocation identifies where a new object is created: a schema, and for indexes also a table.
type objectLocation struct {
label string
schemaIndex int
tableIndex int
}
// objectRow is one existing object shown in an object list.
type objectRow struct {
cells []string
schemaIndex int
tableIndex int
at int // position within the schema slice (views, sequences, scripts)
name string // map key (indexes)
}
// objectKind describes how a kind of schema object is listed and edited.
type objectKind struct {
page string
title string
singular string
headers []string
rows func() []objectRow
locations func() []objectLocation
// buildForm adds the editable fields to the form for row (nil when creating) and
// returns a function that validates and saves the values at the given location.
buildForm func(form *tview.Form, row *objectRow) func(loc objectLocation) error
remove func(row objectRow) bool
}
func (se *SchemaEditor) schemaLocations() []objectLocation {
locs := make([]objectLocation, 0, len(se.db.Schemas))
for si, s := range se.db.Schemas {
locs = append(locs, objectLocation{label: s.Name, schemaIndex: si, tableIndex: -1})
}
return locs
}
func (se *SchemaEditor) tableLocations() []objectLocation {
locs := make([]objectLocation, 0)
for si, s := range se.db.Schemas {
for ti, t := range s.Tables {
locs = append(locs, objectLocation{label: s.Name + "." + t.Name, schemaIndex: si, tableIndex: ti})
}
}
return locs
}
func (se *SchemaEditor) indexKind() objectKind {
return objectKind{
page: "indexes",
title: "Manage Indexes",
singular: "Index",
headers: []string{"Name", "Schema", "Table", "Type", "Unique", "Columns"},
locations: se.tableLocations,
rows: func() []objectRow {
var rows []objectRow
for si, s := range se.db.Schemas {
for ti, t := range s.Tables {
for _, name := range sortedKeys(t.Indexes) {
idx := t.Indexes[name]
rows = append(rows, objectRow{
cells: []string{idx.Name, s.Name, t.Name, idx.Type, strconv.FormatBool(idx.Unique), strings.Join(idx.Columns, ",")},
schemaIndex: si, tableIndex: ti, name: name,
})
}
}
}
return rows
},
buildForm: func(form *tview.Form, row *objectRow) func(objectLocation) error {
idx := models.InitIndex("", "", "")
idx.Type = "btree"
if row != nil {
idx = se.db.Schemas[row.schemaIndex].Tables[row.tableIndex].Indexes[row.name]
}
name, columns, typ, where := idx.Name, strings.Join(idx.Columns, ", "), idx.Type, idx.Where
unique := idx.Unique
form.AddInputField("Name", name, 40, nil, func(v string) { name = v })
form.AddInputField("Columns (comma separated)", columns, 50, nil, func(v string) { columns = v })
form.AddInputField("Type", typ, 20, nil, func(v string) { typ = v })
form.AddCheckbox("Unique", unique, func(v bool) { unique = v })
form.AddInputField("Where", where, 50, nil, func(v string) { where = v })
return func(loc objectLocation) error {
oldName := ""
if row != nil {
oldName = row.name
}
next := *idx
next.Name, next.Columns, next.Type, next.Unique, next.Where = name, splitList(columns), typ, unique, where
return se.SaveIndex(loc.schemaIndex, loc.tableIndex, oldName, &next)
}
},
remove: func(r objectRow) bool { return se.DeleteIndex(r.schemaIndex, r.tableIndex, r.name) },
}
}
func (se *SchemaEditor) viewKind() objectKind {
return objectKind{
page: "views",
title: "Manage Views",
singular: "View",
headers: []string{"Name", "Schema", "Description"},
locations: se.schemaLocations,
rows: func() []objectRow {
var rows []objectRow
for si, s := range se.db.Schemas {
for i, v := range s.Views {
rows = append(rows, objectRow{cells: []string{v.Name, s.Name, v.Description}, schemaIndex: si, at: i})
}
}
return rows
},
buildForm: func(form *tview.Form, row *objectRow) func(objectLocation) error {
view := models.InitView("", "")
at := -1
if row != nil {
view, at = se.db.Schemas[row.schemaIndex].Views[row.at], row.at
}
name, desc, def := view.Name, view.Description, view.Definition
form.AddInputField("Name", name, 40, nil, func(v string) { name = v })
form.AddInputField("Description", desc, 50, nil, func(v string) { desc = v })
form.AddTextArea("Definition (SQL)", def, 60, 8, 0, func(v string) { def = v })
return func(loc objectLocation) error {
next := *view
next.Name, next.Description, next.Definition = name, desc, def
return se.SaveView(loc.schemaIndex, at, &next)
}
},
remove: func(r objectRow) bool { return se.DeleteView(r.schemaIndex, r.at) },
}
}
func (se *SchemaEditor) sequenceKind() objectKind {
return objectKind{
page: "sequences",
title: "Manage Sequences",
singular: "Sequence",
headers: []string{"Name", "Schema", "Start", "Increment", "Cycle", "Description"},
locations: se.schemaLocations,
rows: func() []objectRow {
var rows []objectRow
for si, s := range se.db.Schemas {
for i, q := range s.Sequences {
rows = append(rows, objectRow{
cells: []string{q.Name, s.Name, strconv.FormatInt(q.StartValue, 10), strconv.FormatInt(q.IncrementBy, 10), strconv.FormatBool(q.Cycle), q.Description}, schemaIndex: si, at: i,
})
}
}
return rows
},
buildForm: func(form *tview.Form, row *objectRow) func(objectLocation) error {
seq := models.InitSequence("", "")
at := -1
if row != nil {
seq, at = se.db.Schemas[row.schemaIndex].Sequences[row.at], row.at
}
name, desc := seq.Name, seq.Description
start, incr := strconv.FormatInt(seq.StartValue, 10), strconv.FormatInt(seq.IncrementBy, 10)
minV, maxV := strconv.FormatInt(seq.MinValue, 10), strconv.FormatInt(seq.MaxValue, 10)
cycle := seq.Cycle
form.AddInputField("Name", name, 40, nil, func(v string) { name = v })
form.AddInputField("Description", desc, 50, nil, func(v string) { desc = v })
form.AddInputField("Start", start, 20, nil, func(v string) { start = v })
form.AddInputField("Increment", incr, 20, nil, func(v string) { incr = v })
form.AddInputField("Min (0 = none)", minV, 20, nil, func(v string) { minV = v })
form.AddInputField("Max (0 = none)", maxV, 20, nil, func(v string) { maxV = v })
form.AddCheckbox("Cycle", cycle, func(v bool) { cycle = v })
return func(loc objectLocation) error {
next := *seq
next.Name, next.Description, next.Cycle = name, desc, cycle
for _, f := range []struct {
label string
text string
dst *int64
}{{"start", start, &next.StartValue}, {"increment", incr, &next.IncrementBy}, {"min", minV, &next.MinValue}, {"max", maxV, &next.MaxValue}} {
n, err := strconv.ParseInt(strings.TrimSpace(f.text), 10, 64)
if err != nil {
return fmt.Errorf("%s must be an integer", f.label)
}
*f.dst = n
}
return se.SaveSequence(loc.schemaIndex, at, &next)
}
},
remove: func(r objectRow) bool { return se.DeleteSequence(r.schemaIndex, r.at) },
}
}
func (se *SchemaEditor) scriptKind() objectKind {
return objectKind{
page: "scripts",
title: "Manage Scripts",
singular: "Script",
headers: []string{"Name", "Schema", "Version", "Priority", "Description"},
locations: se.schemaLocations,
rows: func() []objectRow {
var rows []objectRow
for si, s := range se.db.Schemas {
for i, sc := range s.Scripts {
rows = append(rows, objectRow{cells: []string{sc.Name, s.Name, sc.Version, strconv.Itoa(sc.Priority), sc.Description}, schemaIndex: si, at: i})
}
}
return rows
},
buildForm: func(form *tview.Form, row *objectRow) func(objectLocation) error {
script := models.InitScript("")
at := -1
if row != nil {
script, at = se.db.Schemas[row.schemaIndex].Scripts[row.at], row.at
}
name, desc, version, sql, rollback := script.Name, script.Description, script.Version, script.SQL, script.Rollback
priority, runAfter := strconv.Itoa(script.Priority), strings.Join(script.RunAfter, ", ")
form.AddInputField("Name", name, 40, nil, func(v string) { name = v })
form.AddInputField("Description", desc, 50, nil, func(v string) { desc = v })
form.AddInputField("Version", version, 20, nil, func(v string) { version = v })
form.AddInputField("Priority", priority, 10, nil, func(v string) { priority = v })
form.AddInputField("Run after (comma separated)", runAfter, 50, nil, func(v string) { runAfter = v })
form.AddTextArea("SQL", sql, 60, 8, 0, func(v string) { sql = v })
form.AddTextArea("Rollback SQL", rollback, 60, 4, 0, func(v string) { rollback = v })
return func(loc objectLocation) error {
prio, err := strconv.Atoi(strings.TrimSpace(priority))
if err != nil {
return fmt.Errorf("priority must be an integer")
}
next := *script
next.Name, next.Description, next.Version, next.Priority = name, desc, version, prio
next.RunAfter, next.SQL, next.Rollback = splitList(runAfter), sql, rollback
return se.SaveScript(loc.schemaIndex, at, &next)
}
},
remove: func(r objectRow) bool { return se.DeleteScript(r.schemaIndex, r.at) },
}
}
func sortedKeys[V any](m map[string]V) []string {
keys := make([]string, 0, len(m))
for k := range m {
keys = append(keys, k)
}
sort.Strings(keys)
return keys
}
// showObjectList displays all objects of a kind across schemas.
func (se *SchemaEditor) showObjectList(k objectKind) {
flex := tview.NewFlex().SetDirection(tview.FlexRow)
title := tview.NewTextView().SetText("[::b]" + k.title).SetDynamicColors(true).SetTextAlign(tview.AlignCenter)
table := tview.NewTable().SetBorders(true).SetSelectable(true, false).SetFixed(1, 0)
for i, h := range k.headers {
table.SetCell(0, i, tview.NewTableCell(h).SetTextColor(tcell.ColorYellow).SetSelectable(false).SetAlign(tview.AlignLeft))
}
rows := k.rows()
for r, row := range rows {
for c, text := range row.cells {
table.SetCell(r+1, c, tview.NewTableCell(text).SetSelectable(true))
}
}
table.SetTitle(" " + k.title[len("Manage "):] + " ").SetBorder(true).SetTitleAlign(tview.AlignLeft)
back := func() {
se.pages.SwitchToPage("main")
se.pages.RemovePage(k.page)
}
btnNew := tview.NewButton("New " + k.singular + " [n]").SetSelectedFunc(func() { se.showObjectForm(k, nil) })
btnBack := tview.NewButton("Back [b]").SetSelectedFunc(back)
btnNew.SetInputCapture(func(event *tcell.EventKey) *tcell.EventKey {
switch event.Key() {
case tcell.KeyBacktab:
se.app.SetFocus(table)
return nil
case tcell.KeyTab:
se.app.SetFocus(btnBack)
return nil
}
return event
})
btnBack.SetInputCapture(func(event *tcell.EventKey) *tcell.EventKey {
switch event.Key() {
case tcell.KeyBacktab:
se.app.SetFocus(btnNew)
return nil
case tcell.KeyTab:
se.app.SetFocus(table)
return nil
}
return event
})
btnFlex := tview.NewFlex().AddItem(btnNew, 0, 1, true).AddItem(btnBack, 0, 1, false)
table.SetInputCapture(func(event *tcell.EventKey) *tcell.EventKey {
switch {
case event.Key() == tcell.KeyEscape, event.Rune() == 'b':
back()
return nil
case event.Key() == tcell.KeyTab:
se.app.SetFocus(btnNew)
return nil
case event.Key() == tcell.KeyEnter:
if row, _ := table.GetSelection(); row > 0 && row <= len(rows) {
se.showObjectForm(k, &rows[row-1])
return nil
}
case event.Rune() == 'n':
se.showObjectForm(k, nil)
return nil
}
return event
})
flex.AddItem(title, 1, 0, false).AddItem(table, 0, 1, true).AddItem(btnFlex, 1, 0, false)
se.pages.AddPage(k.page, flex, true, true)
}
// showObjectForm shows the create (row == nil) or edit form for an object.
func (se *SchemaEditor) showObjectForm(k objectKind, row *objectRow) {
formPage := k.page + "-form"
form := tview.NewForm()
errView := tview.NewTextView().SetDynamicColors(true)
locs := k.locations()
loc := objectLocation{schemaIndex: -1, tableIndex: -1}
switch {
case row != nil:
loc = objectLocation{schemaIndex: row.schemaIndex, tableIndex: row.tableIndex}
case len(locs) > 0:
loc = locs[0]
labels := make([]string, len(locs))
for i, l := range locs {
labels[i] = l.label
}
form.AddDropDown("Location", labels, 0, func(_ string, i int) { loc = locs[i] })
}
save := k.buildForm(form, row)
closeForm := func() {
se.pages.RemovePage(formPage)
se.pages.RemovePage(k.page)
se.showObjectList(k)
}
form.AddButton("Save", func() {
if err := save(loc); err != nil {
errView.SetText("[red]" + tview.Escape(err.Error()))
return
}
closeForm()
})
if row != nil {
form.AddButton("Delete", func() {
modal := tview.NewModal().
SetText(fmt.Sprintf("Delete %s '%s'? This action cannot be undone.", strings.ToLower(k.singular), row.cells[0])).
AddButtons([]string{"Cancel", "Delete"}).
SetDoneFunc(func(_ int, label string) {
se.pages.RemovePage(formPage + "-delete")
if label == "Delete" {
k.remove(*row)
closeForm()
}
})
se.pages.AddAndSwitchToPage(formPage+"-delete", modal, true)
})
}
form.AddButton("Back", closeForm)
verb := "New"
if row != nil {
verb = "Edit"
}
form.SetBorder(true).SetTitle(" " + verb + " " + k.singular + " ").SetTitleAlign(tview.AlignLeft)
form.SetInputCapture(func(event *tcell.EventKey) *tcell.EventKey {
if event.Key() == tcell.KeyEscape {
se.showExitConfirmation(formPage, k.page)
return nil
}
return event
})
if len(locs) == 0 && row == nil {
errView.SetText("[red]No schema/table available. Create one first.")
}
flex := tview.NewFlex().SetDirection(tview.FlexRow).AddItem(form, 0, 1, true).AddItem(errView, 1, 0, false)
se.pages.AddPage(formPage, flex, true, true)
}
// showDomainTables lists the tables assigned to a domain and allows assigning/unassigning.
func (se *SchemaEditor) showDomainTables(domainIndex int) {
if domainIndex < 0 || domainIndex >= len(se.db.Domains) {
return
}
domain := se.db.Domains[domainIndex]
page := "domain-tables"
list := tview.NewList().ShowSecondaryText(true)
refresh := func() {
se.pages.RemovePage(page)
se.showDomainTables(domainIndex)
}
for _, dt := range domain.Tables {
dt := dt
list.AddItem(dt.SchemaName+"."+dt.TableName, "Enter to remove from domain", 0, func() {
se.UnassignTableFromDomain(domainIndex, dt.SchemaName, dt.TableName)
refresh()
})
}
list.AddItem("[Assign Table]", "Add a table to this domain", 'a', func() {
se.showAssignDomainTable(domainIndex, refresh)
})
list.AddItem("[Back]", "Return to domain", 'b', func() {
se.pages.RemovePage(page)
})
list.SetBorder(true).SetTitle(" Domain " + domain.Name + " - Tables ").SetTitleAlign(tview.AlignLeft)
list.SetInputCapture(func(event *tcell.EventKey) *tcell.EventKey {
if event.Key() == tcell.KeyEscape {
se.pages.RemovePage(page)
return nil
}
return event
})
se.pages.AddPage(page, list, true, true)
}
// showAssignDomainTable shows a form to pick a table not yet in the domain.
func (se *SchemaEditor) showAssignDomainTable(domainIndex int, done func()) {
page := "assign-domain-table"
domain := se.db.Domains[domainIndex]
var options []string
var refs []models.DomainTable
for _, s := range se.db.Schemas {
for _, t := range s.Tables {
taken := false
for _, dt := range domain.Tables {
taken = taken || (dt.SchemaName == s.Name && dt.TableName == t.Name)
}
if !taken {
options = append(options, s.Name+"."+t.Name)
refs = append(refs, models.DomainTable{SchemaName: s.Name, TableName: t.Name})
}
}
}
form := tview.NewForm()
selected := 0
form.AddDropDown("Table", options, 0, func(_ string, i int) { selected = i })
form.AddButton("Assign", func() {
if len(refs) > 0 {
_ = se.AssignTableToDomain(domainIndex, refs[selected].SchemaName, refs[selected].TableName)
}
se.pages.RemovePage(page)
done()
})
form.AddButton("Back", func() { se.pages.RemovePage(page) })
form.SetBorder(true).SetTitle(" Assign Table ").SetTitleAlign(tview.AlignLeft)
form.SetInputCapture(func(event *tcell.EventKey) *tcell.EventKey {
if event.Key() == tcell.KeyEscape {
se.pages.RemovePage(page)
return nil
}
return event
})
se.pages.AddPage(page, form, true, true)
}
+3
View File
@@ -129,6 +129,9 @@ type User struct {
- `default` - Default value
- `rel` - Relationship definition
- `type` - Explicit SQL type
- `scanonly` - Excluded from INSERT/UPDATE: `GENERATED ... STORED` columns, and `GENERATED ALWAYS AS IDENTITY` columns that are not the primary key
- `generated` - Marks a `GENERATED ... STORED` column
- `identity` - Marks a `GENERATED ALWAYS AS IDENTITY` column (a primary key gets only this marker, since Bun drops `scanonly` fields from the PK list)
## Type Mapping
+46
View File
@@ -0,0 +1,46 @@
package bun
import (
"strings"
"testing"
"git.warky.dev/wdevs/relspecgo/pkg/models"
)
func TestBuildBunTag_GeneratedColumn(t *testing.T) {
mapper := NewTypeMapper("", "")
generated := models.InitColumn("full_name", "users", "public")
generated.Type = "text"
generated.Generated = true
generated.GenerationExpression = "first || ' ' || last"
plain := models.InitColumn("first", "users", "public")
plain.Type = "text"
tests := []struct {
name string
column *models.Column
want bool
}{
{"generated column is scanonly and marked", generated, true},
{"ordinary column is untouched", plain, false},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
tag := mapper.BuildBunTag(tt.column, nil)
parts := strings.Split(tag, ",")
has := func(s string) bool {
for _, p := range parts {
if p == s {
return true
}
}
return false
}
if has("scanonly") != tt.want || has("generated") != tt.want {
t.Errorf("tag %q: scanonly/generated present = %v/%v, want %v", tag, has("scanonly"), has("generated"), tt.want)
}
})
}
}
+49
View File
@@ -0,0 +1,49 @@
package bun
import (
"strings"
"testing"
"git.warky.dev/wdevs/relspecgo/pkg/models"
)
func TestBuildBunTag_IdentityColumn(t *testing.T) {
mapper := NewTypeMapper("", "")
build := func(pk, identity bool, generation string) *models.Column {
col := models.InitColumn("id", "users", "public")
col.Type = "bigint"
col.IsPrimaryKey = pk
col.Identity = identity
col.IdentityGeneration = generation
return col
}
tests := []struct {
name string
column *models.Column
wantFlag bool
wantMarker bool
}{
{"always identity non-pk is write-blocked", build(false, true, "ALWAYS"), true, true},
{"always identity pk is only marked", build(true, true, "ALWAYS"), false, true},
{"by default identity is writable", build(false, true, "BY DEFAULT"), false, false},
{"non-identity is untouched", build(false, false, ""), false, false},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
tag := mapper.BuildBunTag(tt.column, nil)
has := func(s string) bool {
for _, p := range strings.Split(tag, ",") {
if p == s {
return true
}
}
return false
}
if has("scanonly") != tt.wantFlag || has("identity") != tt.wantMarker {
t.Errorf("tag %q: scanonly/identity = %v/%v, want %v/%v", tag, has("scanonly"), has("identity"), tt.wantFlag, tt.wantMarker)
}
})
}
}
+8 -5
View File
@@ -279,13 +279,16 @@ func (md *ModelData) AddRelationshipField(field *FieldData) {
// formatComment combines description and comment into a single comment string
func formatComment(description, comment string) string {
var result string
if description != "" && comment != "" {
return description + " - " + comment
result = description + " - " + comment
} else if description != "" {
result = description
} else {
result = comment
}
if description != "" {
return description
}
return comment
// Generated Go comments are emitted on a single source line.
return strings.Join(strings.Fields(result), " ")
}
func isStringLikePrimaryKeyType(goType string) bool {
+16
View File
@@ -366,6 +366,22 @@ func (tm *TypeMapper) BuildBunTag(column *models.Column, table *models.Table) st
parts = append(parts, "autoincrement")
}
// GENERATED ALWAYS ... STORED columns are computed by the database and cannot be
// written: scanonly keeps them out of INSERT/UPDATE, generated marks why.
if column.Generated {
parts = append(parts, "scanonly", "generated")
}
// GENERATED ALWAYS AS IDENTITY rejects explicit values. Bun drops scanonly fields
// from the table's PK list, so a primary key only gets the marker; its zero value
// is already sent as DEFAULT via autoincrement.
if column.Identity && strings.EqualFold(column.IdentityGeneration, "ALWAYS") {
if !column.IsPrimaryKey {
parts = append(parts, "scanonly")
}
parts = append(parts, "identity")
}
// Default value
if column.Default != nil {
// Sanitize default value to remove backticks, then quote based on column type
+56
View File
@@ -1,6 +1,8 @@
package bun
import (
"go/parser"
"go/token"
"os"
"path/filepath"
"strings"
@@ -95,6 +97,60 @@ func TestWriter_WriteTable(t *testing.T) {
}
}
func TestWriter_WriteTable_MultilineDescriptionProducesValidGo(t *testing.T) {
table := models.InitTable("city", "info")
table.Description = "Cities and municipalities worldwide.\n\nSPATIAL:\nProximity queries use a GiST index."
table.Columns["id_city"] = &models.Column{Name: "id_city", Type: "integer", IsPrimaryKey: true, NotNull: true}
outputPath := filepath.Join(t.TempDir(), "city.go")
writer := NewWriter(&writers.WriterOptions{OutputPath: outputPath, PackageName: "models"})
if err := writer.WriteTable(table); err != nil {
t.Fatalf("WriteTable() error = %v", err)
}
generated, err := os.ReadFile(outputPath)
if err != nil {
t.Fatalf("failed to read generated code: %v", err)
}
if _, err := parser.ParseFile(token.NewFileSet(), outputPath, generated, parser.AllErrors); err != nil {
t.Fatalf("generated code is invalid Go: %v\n%s", err, generated)
}
if !strings.Contains(string(generated), "// Cities and municipalities worldwide. SPATIAL: Proximity queries use a GiST index.") {
t.Errorf("multiline description was not rendered as a single Go comment:\n%s", generated)
}
}
func TestWriter_WriteTable_HeaderIsVersionFree(t *testing.T) {
table := models.InitTable("users", "public")
table.Columns["id"] = &models.Column{Name: "id", Type: "bigint", IsPrimaryKey: true, NotNull: true, Sequence: 1}
outputPath := filepath.Join(t.TempDir(), "users.go")
writer := NewWriter(&writers.WriterOptions{OutputPath: outputPath, PackageName: "models"})
if err := writer.WriteTable(table); err != nil {
t.Fatalf("WriteTable() error = %v", err)
}
generated, err := os.ReadFile(outputPath)
if err != nil {
t.Fatalf("failed to read generated code: %v", err)
}
src := string(generated)
if _, err := parser.ParseFile(token.NewFileSet(), outputPath, generated, parser.AllErrors); err != nil {
t.Fatalf("generated code is invalid Go: %v\n%s", err, src)
}
// Output must be reproducible: no version or build date in the header.
if strings.Contains(src, "(built: ") || strings.Contains(src, "RelSpec ") {
t.Errorf("generated code still carries a version header\n%s", src)
}
// The "Code generated" marker must sit directly above the package clause.
want := "// Code generated by relspecgo. DO NOT EDIT.\npackage models"
if !strings.Contains(src, want) {
t.Errorf("header not immediately followed by package clause\n%s", src)
}
}
func TestWriter_WriteDatabase_MultiFile(t *testing.T) {
// Create a database with two tables
db := models.InitDatabase("testdb")
+2
View File
@@ -111,6 +111,8 @@ Table table_name {
- `unique` - Unique constraint
- `default: value` - Default value
- `note: 'text'` - Column note
- `note: 'GENERATED ALWAYS AS (expr) STORED'` - Generated column (`Generated`, `GenerationExpression`)
- `note: 'GENERATED ALWAYS|BY DEFAULT AS IDENTITY'` - Identity column (`Identity`, `IdentityGeneration`)
### Relationships
```dbml
@@ -0,0 +1,77 @@
package dbml
import (
"os"
"path/filepath"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"git.warky.dev/wdevs/relspecgo/pkg/models"
"git.warky.dev/wdevs/relspecgo/pkg/readers"
dbmlreader "git.warky.dev/wdevs/relspecgo/pkg/readers/dbml"
"git.warky.dev/wdevs/relspecgo/pkg/writers"
)
func TestGeneratedAndIdentityColumnsRoundTrip(t *testing.T) {
table := models.InitTable("people", "public")
id := models.InitColumn("id", "people", "public")
id.Type = "bigint"
id.IsPrimaryKey = true
id.AutoIncrement = true
id.Identity = true
id.IdentityGeneration = "ALWAYS"
table.Columns["id"] = id
seq := models.InitColumn("seq", "people", "public")
seq.Type = "integer"
seq.Identity = true
seq.IdentityGeneration = "BY DEFAULT"
table.Columns["seq"] = seq
full := models.InitColumn("full_name", "people", "public")
full.Type = "text"
full.Generated = true
full.GenerationExpression = "first || ' ' || last"
table.Columns["full_name"] = full
plain := models.InitColumn("first", "people", "public")
plain.Type = "text"
plain.Comment = "given name"
table.Columns["first"] = plain
schema := models.InitSchema("public")
schema.Tables = append(schema.Tables, table)
db := models.InitDatabase("testdb")
db.Schemas = append(db.Schemas, schema)
out := filepath.Join(t.TempDir(), "people.dbml")
require.NoError(t, NewWriter(&writers.WriterOptions{OutputPath: out}).WriteDatabase(db))
content, err := os.ReadFile(out)
require.NoError(t, err)
assert.Contains(t, string(content), "GENERATED ALWAYS AS (first || \\' \\' || last) STORED")
assert.Contains(t, string(content), "GENERATED ALWAYS AS IDENTITY")
assert.Contains(t, string(content), "GENERATED BY DEFAULT AS IDENTITY")
got, err := dbmlreader.NewReader(&readers.ReaderOptions{FilePath: out}).ReadDatabase()
require.NoError(t, err)
cols := got.Schemas[0].Tables[0].Columns
assert.True(t, cols["id"].Identity)
assert.Equal(t, "ALWAYS", cols["id"].IdentityGeneration)
assert.Empty(t, cols["id"].Comment, "marker note must not become a comment")
assert.True(t, cols["seq"].Identity)
assert.Equal(t, "BY DEFAULT", cols["seq"].IdentityGeneration)
assert.True(t, cols["full_name"].Generated)
assert.Equal(t, "first || ' ' || last", cols["full_name"].GenerationExpression)
assert.Empty(t, cols["full_name"].Comment)
assert.False(t, cols["first"].Generated)
assert.False(t, cols["first"].Identity)
assert.Equal(t, "given name", cols["first"].Comment)
}
+27 -1
View File
@@ -114,6 +114,24 @@ func (w *Writer) schemaToDBML(schema *models.Schema) string {
return sb.String()
}
// columnMarkerNote returns the note marking a column the database computes itself
// (GENERATED ... STORED, or GENERATED ... AS IDENTITY), or "" for an ordinary column.
// The DBML reader turns the note back into the model's Generated/Identity fields.
func columnMarkerNote(column *models.Column) string {
switch {
case column.Generated:
return fmt.Sprintf("GENERATED ALWAYS AS (%s) STORED", column.GenerationExpression)
case column.Identity && column.IdentityGeneration != "":
return fmt.Sprintf("GENERATED %s AS IDENTITY", strings.ToUpper(column.IdentityGeneration))
}
return ""
}
// escapeNote escapes a value for use inside a single-quoted DBML note.
func escapeNote(s string) string {
return strings.ReplaceAll(strings.ReplaceAll(s, `\`, `\\`), `'`, `\'`)
}
// tableToDBML converts a Table to DBML format string
func (w *Writer) tableToDBML(t *models.Table) string {
var sb strings.Builder
@@ -146,6 +164,10 @@ func (w *Writer) tableToDBML(t *models.Table) string {
}
}
if marker := columnMarkerNote(column); marker != "" {
attrs = append(attrs, fmt.Sprintf("note: '%s'", escapeNote(marker)))
}
if len(attrs) > 0 {
fmt.Fprintf(&sb, " [%s]", strings.Join(attrs, ", "))
}
@@ -196,7 +218,11 @@ func (w *Writer) tableToDBML(t *models.Table) string {
note := strings.TrimSpace(t.Description + " " + t.Comment)
if note != "" {
fmt.Fprintf(&sb, "\n Note: '%s'\n", note)
if strings.Contains(note, "\n") {
fmt.Fprintf(&sb, "\n Note: '''\n%s\n '''\n", note)
} else {
fmt.Fprintf(&sb, "\n Note: '%s'\n", note)
}
}
sb.WriteString("}\n")
+17
View File
@@ -59,6 +59,23 @@ func TestWriter_WriteTable(t *testing.T) {
assert.Contains(t, output, "Note: 'User accounts table'")
}
func TestWriter_WriteTable_MultilineNote(t *testing.T) {
table := models.InitTable("cities", "info")
table.Description = "Cities and municipalities worldwide.\n\nSPATIAL:\nProximity queries use a GiST index."
outputPath := filepath.Join(t.TempDir(), "cities.dbml")
writer := NewWriter(&writers.WriterOptions{OutputPath: outputPath})
if err := writer.WriteTable(table); err != nil {
t.Fatalf("WriteTable() error = %v", err)
}
output, err := os.ReadFile(outputPath)
if err != nil {
t.Fatalf("failed to read generated DBML: %v", err)
}
assert.Contains(t, string(output), "Note: '''\nCities and municipalities worldwide.\n\nSPATIAL:\nProximity queries use a GiST index.\n '''")
}
func TestWriter_WriteDatabase_WithRelationships(t *testing.T) {
db := models.InitDatabase("test_db")
schema := models.InitSchema("public")
+1
View File
@@ -113,6 +113,7 @@ export const postsRelations = relations(posts, ({ one }) => ({
## Notes
- Generated columns: `.generatedAlwaysAs(sql`expr`)`; identity columns: `.generatedAlwaysAsIdentity()` / `.generatedByDefaultAsIdentity()`
- Table names and column names are preserved as-is
- Relationships are generated as separate relation definitions
- Constraint actions (CASCADE, etc.) are included in references
@@ -0,0 +1,71 @@
package drizzle
import (
"os"
"path/filepath"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"git.warky.dev/wdevs/relspecgo/pkg/models"
"git.warky.dev/wdevs/relspecgo/pkg/readers"
drizzlereader "git.warky.dev/wdevs/relspecgo/pkg/readers/drizzle"
"git.warky.dev/wdevs/relspecgo/pkg/writers"
)
func TestGeneratedAndIdentityColumnsRoundTrip(t *testing.T) {
table := models.InitTable("people", "public")
id := models.InitColumn("id", "people", "public")
id.Type = "bigint"
id.IsPrimaryKey = true
id.AutoIncrement = true
id.Identity = true
id.IdentityGeneration = "ALWAYS"
table.Columns["id"] = id
seq := models.InitColumn("seq", "people", "public")
seq.Type = "integer"
seq.AutoIncrement = true
seq.Identity = true
seq.IdentityGeneration = "BY DEFAULT"
table.Columns["seq"] = seq
first := models.InitColumn("first", "people", "public")
first.Type = "text"
table.Columns["first"] = first
full := models.InitColumn("full_name", "people", "public")
full.Type = "text"
full.Generated = true
full.GenerationExpression = "first || ' ' || upper(first)"
table.Columns["full_name"] = full
schema := models.InitSchema("public")
schema.Tables = append(schema.Tables, table)
db := models.InitDatabase("testdb")
db.Schemas = append(db.Schemas, schema)
out := filepath.Join(t.TempDir(), "schema.ts")
require.NoError(t, NewWriter(&writers.WriterOptions{OutputPath: out}).WriteDatabase(db))
content, err := os.ReadFile(out)
require.NoError(t, err)
assert.Contains(t, string(content), "generatedAlwaysAs(sql`first || ' ' || upper(first)`)")
assert.Contains(t, string(content), "generatedAlwaysAsIdentity()")
assert.Contains(t, string(content), "generatedByDefaultAsIdentity()")
got, err := drizzlereader.NewReader(&readers.ReaderOptions{FilePath: out}).ReadDatabase()
require.NoError(t, err)
cols := got.Schemas[0].Tables[0].Columns
assert.True(t, cols["id"].Identity)
assert.Equal(t, "ALWAYS", cols["id"].IdentityGeneration)
assert.True(t, cols["seq"].Identity)
assert.Equal(t, "BY DEFAULT", cols["seq"].IdentityGeneration)
assert.True(t, cols["full_name"].Generated)
assert.Equal(t, "first || ' ' || upper(first)", cols["full_name"].GenerationExpression)
assert.False(t, cols["first"].Generated)
assert.False(t, cols["first"].Identity)
}
+9 -5
View File
@@ -2,6 +2,7 @@ package drizzle
import (
"sort"
"strings"
"git.warky.dev/wdevs/relspecgo/pkg/models"
)
@@ -199,13 +200,16 @@ func NewIndexData(index *models.Index, tableVar string, tm *TypeMapper) *IndexDa
// formatComment combines description and comment into a single comment string
func formatComment(description, comment string) string {
var result string
if description != "" && comment != "" {
return description + " - " + comment
result = description + " - " + comment
} else if description != "" {
result = description
} else {
result = comment
}
if description != "" {
return description
}
return comment
// Generated TypeScript comments are emitted on a single source line.
return strings.Join(strings.Fields(result), " ")
}
// joinStrings joins a slice of strings with a separator
+7 -1
View File
@@ -140,7 +140,13 @@ func (tm *TypeMapper) buildColumnModifiers(col *models.Column, table *models.Tab
}
// Default value
if col.AutoIncrement {
if col.Generated && col.GenerationExpression != "" {
// GENERATED ALWAYS AS (expr) STORED
expr := strings.ReplaceAll(col.GenerationExpression, "`", "\\`")
modifiers = append(modifiers, fmt.Sprintf("generatedAlwaysAs(sql`%s`)", expr))
} else if col.Identity && strings.EqualFold(col.IdentityGeneration, "BY DEFAULT") {
modifiers = append(modifiers, "generatedByDefaultAsIdentity()")
} else if col.AutoIncrement || col.Identity {
// For auto-increment, use generatedAlwaysAsIdentity()
modifiers = append(modifiers, "generatedAlwaysAsIdentity()")
} else if col.Default != nil {
+8
View File
@@ -203,6 +203,14 @@ The writer automatically generates relationship fields:
- Relationship field names use 3-letter prefixes
- Includes proper `gorm` tags with `foreignKey` and `references`
## Read-only Columns
| Column | Tags |
|---|---|
| `GENERATED ... STORED` | `<-:false;generated` |
| `GENERATED ALWAYS AS IDENTITY`, not PK | `<-:false;identity` |
| `GENERATED ALWAYS AS IDENTITY`, PK | `identity` only |
## Notes
- Model names are prefixed with "Model" (e.g., `ModelUser`)
+45
View File
@@ -0,0 +1,45 @@
package gorm
import (
"strings"
"testing"
"git.warky.dev/wdevs/relspecgo/pkg/models"
)
func TestBuildGormTag_GeneratedColumn(t *testing.T) {
mapper := NewTypeMapper("")
generated := models.InitColumn("full_name", "users", "public")
generated.Type = "text"
generated.Generated = true
plain := models.InitColumn("first", "users", "public")
plain.Type = "text"
tests := []struct {
name string
column *models.Column
want bool
}{
{"generated column is write-disabled and marked", generated, true},
{"ordinary column is untouched", plain, false},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
tag := mapper.BuildGormTag(tt.column, nil)
parts := strings.Split(tag, ";")
has := func(s string) bool {
for _, p := range parts {
if p == s {
return true
}
}
return false
}
if has("<-:false") != tt.want || has("generated") != tt.want {
t.Errorf("tag %q: <-:false/generated present = %v/%v, want %v", tag, has("<-:false"), has("generated"), tt.want)
}
})
}
}
+49
View File
@@ -0,0 +1,49 @@
package gorm
import (
"strings"
"testing"
"git.warky.dev/wdevs/relspecgo/pkg/models"
)
func TestBuildGormTag_IdentityColumn(t *testing.T) {
mapper := NewTypeMapper("")
build := func(pk, identity bool, generation string) *models.Column {
col := models.InitColumn("id", "users", "public")
col.Type = "bigint"
col.IsPrimaryKey = pk
col.Identity = identity
col.IdentityGeneration = generation
return col
}
tests := []struct {
name string
column *models.Column
wantFlag bool
wantMarker bool
}{
{"always identity non-pk is write-blocked", build(false, true, "ALWAYS"), true, true},
{"always identity pk is only marked", build(true, true, "ALWAYS"), false, true},
{"by default identity is writable", build(false, true, "BY DEFAULT"), false, false},
{"non-identity is untouched", build(false, false, ""), false, false},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
tag := mapper.BuildGormTag(tt.column, nil)
has := func(s string) bool {
for _, p := range strings.Split(tag, ";") {
if p == s {
return true
}
}
return false
}
if has("<-:false") != tt.wantFlag || has("identity") != tt.wantMarker {
t.Errorf("tag %q: <-:false/identity = %v/%v, want %v/%v", tag, has("<-:false"), has("identity"), tt.wantFlag, tt.wantMarker)
}
})
}
}
+9 -5
View File
@@ -192,13 +192,17 @@ func (md *ModelData) AddRelationshipField(field *FieldData) {
// formatComment combines description and comment into a single comment string
func formatComment(description, comment string) string {
var result string
if description != "" && comment != "" {
return description + " - " + comment
result = description + " - " + comment
} else if description != "" {
result = description
} else {
result = comment
}
if description != "" {
return description
}
return comment
// Generated Go comments are emitted on a single source line. Collapse
// multiline DBML notes so the remaining lines cannot become invalid Go.
return strings.Join(strings.Fields(result), " ")
}
func isStringLikePrimaryKeyType(goType string) bool {
+15
View File
@@ -401,6 +401,21 @@ func (tm *TypeMapper) BuildGormTag(column *models.Column, table *models.Table) s
parts = append(parts, "autoIncrement")
}
// GENERATED ALWAYS ... STORED columns are computed by the database and cannot be
// written: disable write permission so GORM leaves them out of INSERT/UPDATE, and mark why.
if column.Generated {
parts = append(parts, "<-:false", "generated")
}
// GENERATED ALWAYS AS IDENTITY rejects explicit values. A primary key only gets the
// marker so it keeps its normal create/return behavior.
if column.Identity && strings.EqualFold(column.IdentityGeneration, "ALWAYS") {
if !column.IsPrimaryKey {
parts = append(parts, "<-:false")
}
parts = append(parts, "identity")
}
// Not null (skip if primary key, as it's implied)
if column.NotNull && !column.IsPrimaryKey {
parts = append(parts, "not null")
+65
View File
@@ -1,6 +1,8 @@
package gorm
import (
"go/parser"
"go/token"
"os"
"path/filepath"
"strings"
@@ -87,6 +89,69 @@ func TestWriter_WriteTable(t *testing.T) {
}
}
func TestWriter_WriteTable_MultilineDescriptionProducesValidGo(t *testing.T) {
table := models.InitTable("cities", "info")
table.Description = "Cities and municipalities worldwide.\n\nSPATIAL:\nProximity queries use a GiST index."
table.Columns["id"] = &models.Column{Name: "id", Type: "integer", IsPrimaryKey: true, NotNull: true}
outputPath := filepath.Join(t.TempDir(), "cities.go")
writer := NewWriter(&writers.WriterOptions{OutputPath: outputPath, PackageName: "models"})
if err := writer.WriteTable(table); err != nil {
t.Fatalf("WriteTable() error = %v", err)
}
generated, err := os.ReadFile(outputPath)
if err != nil {
t.Fatalf("failed to read generated code: %v", err)
}
if _, err := parser.ParseFile(token.NewFileSet(), outputPath, generated, parser.AllErrors); err != nil {
t.Fatalf("generated code is invalid Go: %v\n%s", err, generated)
}
if !strings.Contains(string(generated), "// Cities and municipalities worldwide. SPATIAL: Proximity queries use a GiST index.") {
t.Errorf("multiline description was not rendered as a single Go comment:\n%s", generated)
}
}
func TestWriter_WriteTable_ColumnOrderFollowsSequence(t *testing.T) {
// Column.Sequence carries the source (e.g. DBML) declaration order; the
// generated struct fields must follow it, not fall back to alphabetical.
table := models.InitTable("widget", "public")
table.Columns["zeta"] = &models.Column{Name: "zeta", Type: "varchar", Length: 50, Sequence: 1}
table.Columns["alpha"] = &models.Column{Name: "alpha", Type: "bigint", NotNull: true, IsPrimaryKey: true, AutoIncrement: true, Sequence: 2}
table.Columns["mid_field"] = &models.Column{Name: "mid_field", Type: "integer", Sequence: 3}
table.Columns["beta"] = &models.Column{Name: "beta", Type: "varchar", Length: 100, Sequence: 4}
outputPath := filepath.Join(t.TempDir(), "widget.go")
writer := NewWriter(&writers.WriterOptions{OutputPath: outputPath, PackageName: "models"})
if err := writer.WriteTable(table); err != nil {
t.Fatalf("WriteTable() error = %v", err)
}
generated, err := os.ReadFile(outputPath)
if err != nil {
t.Fatalf("failed to read generated code: %v", err)
}
src := string(generated)
if _, err := parser.ParseFile(token.NewFileSet(), outputPath, generated, parser.AllErrors); err != nil {
t.Fatalf("generated code is invalid Go: %v\n%s", err, src)
}
positions := make([]int, 0, 4)
for _, field := range []string{"Zeta ", "Alpha ", "MidField ", "Beta "} {
idx := strings.Index(src, "\t"+field)
if idx < 0 {
t.Fatalf("field %q missing from generated struct:\n%s", field, src)
}
positions = append(positions, idx)
}
for i := 1; i < len(positions); i++ {
if positions[i] <= positions[i-1] {
t.Errorf("struct fields not in Sequence order (want zeta, alpha, mid_field, beta):\n%s", src)
break
}
}
}
func TestWriter_WriteDatabase_MultiFile(t *testing.T) {
// Create a database with two tables
db := models.InitDatabase("testdb")
+2 -1
View File
@@ -127,4 +127,5 @@ go test ./pkg/writers/mssql/...
- Views are not currently supported in the writer
- Sequences are not supported (MSSQL uses IDENTITY instead)
- Partitioning and advanced features are not supported
- Generated DDL assumes no triggers or computed columns
- Generated DDL assumes no triggers
- Generated columns are written as computed columns `[col] AS (expr) PERSISTED` (no type, no DEFAULT); identity columns as `IDENTITY(1,1)`
+11 -1
View File
@@ -187,6 +187,16 @@ func (w *Writer) writeCreateTable(schema *models.Schema, table *models.Table) er
func (w *Writer) generateColumnDefinition(col *models.Column) string {
parts := []string{fmt.Sprintf("[%s]", col.Name)}
// Computed column: "name AS (expr) PERSISTED" has no data type and cannot carry
// IDENTITY or a DEFAULT. PERSISTED mirrors the stored semantics of Column.Generated.
if col.Generated && col.GenerationExpression != "" {
parts = append(parts, fmt.Sprintf("AS (%s) PERSISTED", col.GenerationExpression))
if col.NotNull {
parts = append(parts, "NOT NULL")
}
return strings.Join(parts, " ")
}
// Type with length/precision
baseType := mssql.ConvertCanonicalToMSSQL(col.Type)
typeStr := baseType
@@ -215,7 +225,7 @@ func (w *Writer) generateColumnDefinition(col *models.Column) string {
parts = append(parts, typeStr)
// IDENTITY for auto-increment
if col.AutoIncrement {
if col.AutoIncrement || col.Identity {
parts = append(parts, "IDENTITY(1,1)")
}
+16
View File
@@ -204,3 +204,19 @@ func TestWriteComments(t *testing.T) {
assert.Contains(t, output, "User accounts table")
assert.Contains(t, output, "Primary key")
}
func TestGenerateColumnDefinition_ComputedAndIdentity(t *testing.T) {
writer := NewWriter(&writers.WriterOptions{})
computed := &models.Column{
Name: "full_name",
Type: "string",
Generated: true,
GenerationExpression: "[first] + ' ' + [last]",
Default: "ignored",
}
assert.Equal(t, "[full_name] AS ([first] + ' ' + [last]) PERSISTED", writer.generateColumnDefinition(computed))
identity := &models.Column{Name: "id", Type: "int", NotNull: true, Identity: true, IdentityGeneration: "ALWAYS"}
assert.Equal(t, "[id] INT IDENTITY(1,1) NOT NULL", writer.generateColumnDefinition(identity))
}
+171
View File
@@ -0,0 +1,171 @@
package mysql
import (
"context"
"database/sql"
"fmt"
"io"
"os"
"sort"
"strings"
_ "github.com/go-sql-driver/mysql"
"git.warky.dev/wdevs/relspecgo/pkg/mariadb"
"git.warky.dev/wdevs/relspecgo/pkg/models"
"git.warky.dev/wdevs/relspecgo/pkg/writers"
)
type Writer struct {
options *writers.WriterOptions
writer io.Writer
}
func NewWriter(options *writers.WriterOptions) *Writer { return &Writer{options: options} }
func (w *Writer) WriteDatabase(db *models.Database) error {
if w.options == nil {
return fmt.Errorf("writer options are required")
}
if conn, ok := w.options.Metadata["connection_string"].(string); ok && conn != "" {
return w.execute(db, conn)
}
if w.writer == nil {
if w.options.OutputPath != "" {
f, err := os.Create(w.options.OutputPath)
if err != nil {
return err
}
defer f.Close()
w.writer = f
} else {
w.writer = os.Stdout
}
}
return w.writeDatabaseDDL(db)
}
func (w *Writer) writeDatabaseDDL(db *models.Database) error {
if _, err := fmt.Fprintf(w.writer, "-- MySQL Database Schema\n-- Database: %s\n-- Generated by RelSpec\n\n", db.Name); err != nil {
return err
}
for _, s := range db.Schemas {
if err := w.WriteSchema(s); err != nil {
return err
}
}
return nil
}
func (w *Writer) WriteSchema(s *models.Schema) error {
for _, t := range s.Tables {
if err := w.writeTable(s, t); err != nil {
return err
}
}
return nil
}
func (w *Writer) WriteTable(t *models.Table) error {
if w.writer == nil {
w.writer = os.Stdout
}
return w.writeTable(nil, t)
}
func (w *Writer) writeTable(s *models.Schema, t *models.Table) error {
name := t.Name
if s != nil {
name = fmt.Sprintf("%s.%s", quote(s.Name), quote(t.Name))
} else {
name = quote(name)
}
cols := make([]*models.Column, 0, len(t.Columns))
for _, c := range t.Columns {
cols = append(cols, c)
}
sort.Slice(cols, func(i, j int) bool { return cols[i].Sequence < cols[j].Sequence })
defs := []string{}
pk := []string{}
for _, c := range cols {
d := fmt.Sprintf(" %s %s", quote(c.Name), mariadb.ConvertCanonicalToMariaDB(c.Type))
if c.Length > 0 && strings.EqualFold(c.Type, "string") {
d = fmt.Sprintf(" %s VARCHAR(%d)", quote(c.Name), c.Length)
}
if c.NotNull {
d += " NOT NULL"
}
if c.AutoIncrement {
d += " AUTO_INCREMENT"
}
if c.Default != nil {
d += fmt.Sprintf(" DEFAULT %s", writers.QuoteDefaultValue(fmt.Sprint(c.Default), c.Type))
}
defs = append(defs, d)
if c.IsPrimaryKey {
pk = append(pk, quote(c.Name))
}
}
for _, c := range t.Constraints {
if c.Type == models.PrimaryKeyConstraint {
pk = nil
for _, n := range c.Columns {
pk = append(pk, quote(n))
}
break
}
}
if len(pk) > 0 {
defs = append(defs, " PRIMARY KEY ("+strings.Join(pk, ", ")+")")
}
for _, c := range t.Constraints {
if c.Type == models.UniqueConstraint {
defs = append(defs, fmt.Sprintf(" CONSTRAINT %s UNIQUE (%s)", quote(c.Name), quoted(c.Columns)))
}
}
sql := fmt.Sprintf("CREATE TABLE IF NOT EXISTS %s (\n%s\n) ENGINE=InnoDB;\n\n", name, strings.Join(defs, ",\n"))
if _, err := io.WriteString(w.writer, sql); err != nil {
return err
}
return nil
}
func (w *Writer) execute(dbm *models.Database, conn string) error {
var b strings.Builder
old := w.writer
w.writer = &b
if err := w.writeDatabaseDDL(dbm); err != nil {
w.writer = old
return err
}
w.writer = old
db, err := sql.Open("mysql", conn)
if err != nil {
return fmt.Errorf("failed to connect: %w", err)
}
defer db.Close()
for _, stmt := range strings.Split(b.String(), ";\n") {
stmt = stripComments(strings.TrimSpace(stmt))
if stmt == "" {
continue
}
if _, err := db.ExecContext(context.Background(), stmt); err != nil {
return fmt.Errorf("failed to execute SQL: %w", err)
}
}
return nil
}
func stripComments(sqlText string) string {
lines := strings.Split(sqlText, "\n")
kept := lines[:0]
for _, line := range lines {
if !strings.HasPrefix(strings.TrimSpace(line), "--") {
kept = append(kept, line)
}
}
return strings.TrimSpace(strings.Join(kept, "\n"))
}
func quote(s string) string { return "`" + strings.ReplaceAll(s, "`", "``") + "`" }
func quoted(xs []string) string {
out := make([]string, len(xs))
for i, x := range xs {
out[i] = quote(x)
}
return strings.Join(out, ", ")
}

Some files were not shown because too many files have changed in this diff Show More