Compare commits
66
Commits
d6d0200938
...
master
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
08e1417393 | ||
|
|
70282fff73 | ||
|
|
43265dac0f | ||
|
|
66b90ca54b | ||
|
|
47108809aa | ||
|
|
720476fd6e | ||
|
|
572d03fe42 | ||
|
|
f1b9079b2d | ||
|
|
bb671c3680 | ||
|
|
bc8284db25 | ||
|
|
29e747393d | ||
|
|
df980a3434 | ||
|
|
778379538b | ||
|
|
7a9219b6e3 | ||
|
|
fd9c37cd25 | ||
|
|
0235a28add | ||
|
|
d961536186 | ||
|
|
d36806047b | ||
|
|
948419ffd3 | ||
|
|
b38f53c603 | ||
|
|
ccba53c494 | ||
|
|
53327b9a5a | ||
|
|
734b14d48d | ||
|
|
938f0ed51f | ||
|
|
6e2e7eb19e | ||
|
|
82e86cdf39 | ||
|
|
b7ae00ab52 | ||
|
|
8f8664357e | ||
|
|
7bd61a21b5 | ||
|
|
6f6b9834ca | ||
|
|
9cc10715e3 | ||
|
|
bbab5ce936 | ||
|
|
275424c605 | ||
|
|
b6c0cd3d1b | ||
|
|
d7d1d99ebc | ||
|
|
43849324ce | ||
|
|
e1df3b7cee | ||
|
|
f7b5d5f054 | ||
|
|
99fc4b0944 | ||
|
|
278d488363 | ||
|
|
2f69205aa0 | ||
|
|
b91985c493 | ||
|
|
80a3453233 | ||
|
|
ade71b75fa | ||
|
|
799b7feb53 | ||
|
|
eb40201b59 | ||
|
|
1461f8f69f | ||
|
|
ea70e19a46 | ||
|
|
52f4642d97 | ||
|
|
a84592374c | ||
|
|
ca226e83df | ||
|
|
19a2cc1fe3 | ||
|
|
58e46e5b59 | ||
|
|
161ac317f0 | ||
|
|
ee5c009234 | ||
|
|
9e9a17d578 | ||
|
|
7b628b888c | ||
|
|
ce3b615b0a | ||
|
|
84a6b31873 | ||
|
|
f968e3d4a6 | ||
|
|
3d57c947cd | ||
|
|
cdd066dafe | ||
|
|
5c25f333b7 | ||
|
|
77e6e72f5a | ||
|
|
4d4bc09b86 | ||
|
|
96281c9f03 |
@@ -20,12 +20,34 @@ jobs:
|
||||
with:
|
||||
go-version-file: go.mod
|
||||
|
||||
- name: go vet
|
||||
run: go vet ./...
|
||||
|
||||
- name: Install lint tools
|
||||
run: |
|
||||
go install github.com/golangci/golangci-lint/v2/cmd/golangci-lint@latest
|
||||
go install honnef.co/go/tools/cmd/staticcheck@latest
|
||||
go install golang.org/x/vuln/cmd/govulncheck@latest
|
||||
echo "$(go env GOPATH)/bin" >> "$GITHUB_PATH"
|
||||
|
||||
- name: gofumpt (golangci-lint fmt)
|
||||
run: |
|
||||
diff=$(golangci-lint fmt --diff)
|
||||
if [ -n "$diff" ]; then
|
||||
echo "$diff"
|
||||
echo "Formatting issues found. Run: make fmt"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
- name: staticcheck
|
||||
run: staticcheck ./...
|
||||
|
||||
- name: govulncheck
|
||||
run: govulncheck ./...
|
||||
|
||||
- name: Test
|
||||
run: go test ./...
|
||||
|
||||
- name: Lint
|
||||
run: go vet ./...
|
||||
|
||||
release:
|
||||
needs: test
|
||||
runs-on: ubuntu-latest
|
||||
@@ -41,6 +63,8 @@ jobs:
|
||||
- name: Build release binaries
|
||||
run: |
|
||||
VERSION="${{ github.event.inputs.tag || github.ref_name }}"
|
||||
BUILD_DATE="$(date -u +"%Y-%m-%d %H:%M:%S UTC")"
|
||||
LDFLAGS="-X 'git.warky.dev/wdevs/relspecgo/pkg/buildinfo.Version=${VERSION}' -X 'git.warky.dev/wdevs/relspecgo/pkg/buildinfo.BuildDate=${BUILD_DATE}'"
|
||||
for target in "linux/amd64" "linux/arm64" "darwin/amd64" "darwin/arm64" "windows/amd64"; do
|
||||
GOOS="${target%/*}"
|
||||
GOARCH="${target#*/}"
|
||||
@@ -49,7 +73,7 @@ jobs:
|
||||
NAME="relspec-${GOOS}-${GOARCH}${EXT}"
|
||||
GOOS="$GOOS" GOARCH="$GOARCH" go build \
|
||||
-trimpath \
|
||||
-ldflags "-X git.warky.dev/wdevs/relspecgo/cmd/relspec.version=${VERSION}" \
|
||||
-ldflags "$LDFLAGS" \
|
||||
-o "$NAME" ./cmd/relspec
|
||||
echo "Built $NAME"
|
||||
done
|
||||
@@ -212,11 +236,12 @@ jobs:
|
||||
run: |
|
||||
VERSION="${{ github.event.inputs.tag || github.ref_name }}"
|
||||
PKGVER="${VERSION#v}"
|
||||
BUILD_DATE="$(date -u +"%Y-%m-%d %H:%M:%S UTC")"
|
||||
|
||||
for GOARCH in amd64 arm64; do
|
||||
GOOS=linux GOARCH=$GOARCH go build \
|
||||
-trimpath \
|
||||
-ldflags "-X git.warky.dev/wdevs/relspecgo/cmd/relspec.version=${PKGVER}" \
|
||||
-ldflags "-X 'git.warky.dev/wdevs/relspecgo/pkg/buildinfo.Version=v${PKGVER}' -X 'git.warky.dev/wdevs/relspecgo/pkg/buildinfo.BuildDate=${BUILD_DATE}'" \
|
||||
-o relspec ./cmd/relspec
|
||||
|
||||
PKGDIR="relspec_${PKGVER}_${GOARCH}"
|
||||
@@ -287,7 +312,13 @@ jobs:
|
||||
rockylinux:9 \
|
||||
bash -lc "
|
||||
set -euo pipefail
|
||||
dnf install -y rpm-build git &&
|
||||
# Avoid transient/out-of-sync mirrors while bootstrapping the RPM builder.
|
||||
sed -i \
|
||||
-e 's|^mirrorlist=|#mirrorlist=|' \
|
||||
-e 's|^#baseurl=http://dl.rockylinux.org|baseurl=https://dl.rockylinux.org|' \
|
||||
/etc/yum.repos.d/rocky*.repo
|
||||
dnf clean all
|
||||
dnf -y --setopt=timeout=60 --setopt=retries=5 install rpm-build git
|
||||
curl -fsSL https://go.dev/dl/go\${GO_VER}.linux-amd64.tar.gz | tar -C /usr/local -xz &&
|
||||
export PATH=\$PATH:/usr/local/go/bin &&
|
||||
mkdir -p ~/rpmbuild/{BUILD,BUILDROOT,RPMS,SOURCES,SPECS,SRPMS} &&
|
||||
|
||||
+5
-3
@@ -1,7 +1,7 @@
|
||||
{
|
||||
"formatters": {
|
||||
"enable": [
|
||||
"gofmt",
|
||||
"gofumpt",
|
||||
"goimports"
|
||||
],
|
||||
"exclusions": {
|
||||
@@ -13,8 +13,10 @@
|
||||
]
|
||||
},
|
||||
"settings": {
|
||||
"gofmt": {
|
||||
"simplify": true
|
||||
"gofumpt": {
|
||||
"extra": {
|
||||
"group-params": true
|
||||
}
|
||||
},
|
||||
"goimports": {
|
||||
"local-prefixes": [
|
||||
|
||||
+59
-113
@@ -1,145 +1,91 @@
|
||||
# Contributing to RelSpec
|
||||
|
||||
Thank you for your interest in contributing to RelSpec.
|
||||
## Setup
|
||||
|
||||
## Development Setup
|
||||
- Go 1.25+ (see `go.mod`), Git
|
||||
- Optional: golangci-lint, Docker/Podman (PostgreSQL integration tests)
|
||||
|
||||
### Prerequisites
|
||||
- Go 1.21 or higher
|
||||
- Git
|
||||
- (Optional) golangci-lint for linting
|
||||
- (Optional) Docker for database testing
|
||||
|
||||
### Getting Started
|
||||
|
||||
1. Clone the repository:
|
||||
```bash
|
||||
git clone https://github.com/wdevs/relspecgo.git
|
||||
git clone git@git.warky.dev:wdevs/relspecgo.git
|
||||
cd relspecgo
|
||||
make deps
|
||||
make build # outputs build/relspec
|
||||
```
|
||||
|
||||
2. Install dependencies:
|
||||
```bash
|
||||
go mod download
|
||||
```
|
||||
## Make Targets
|
||||
|
||||
3. Run tests:
|
||||
```bash
|
||||
go test ./...
|
||||
```
|
||||
| Target | Purpose |
|
||||
|-------------------------|------------------------------------------------------|
|
||||
| `make test` | Unit tests (race detection, coverage) |
|
||||
| `make test-integration` | Integration tests (needs `RELSPEC_TEST_PG_CONN`) |
|
||||
| `make docker-test` | PostgreSQL integration tests via Docker/Podman |
|
||||
| `make lint` | golangci-lint |
|
||||
| `make fmt` / `fmt-check`| gofumpt + goimports |
|
||||
| `make check` | vet, fmt-check, staticcheck, govulncheck |
|
||||
| `make coverage` | Coverage report |
|
||||
|
||||
4. Build the project:
|
||||
```bash
|
||||
go build -o relspec ./cmd/relspec
|
||||
```
|
||||
Single test: `go test -run TestName ./pkg/readers/dbml`
|
||||
|
||||
## Project Structure
|
||||
## Layout
|
||||
|
||||
```
|
||||
relspecgo/
|
||||
├── cmd/ # CLI application entry point
|
||||
├── pkg/
|
||||
│ ├── readers/ # Input format readers (XML, JSON, DCTX, DB, GORM, Bun)
|
||||
│ ├── writers/ # Output format writers (GORM, Bun, JSON, YAML)
|
||||
│ ├── models/ # Internal data models for relations
|
||||
│ └── transform/ # Transformation and validation logic
|
||||
├── examples/ # Usage examples and sample files
|
||||
├── tests/ # Integration tests
|
||||
└── .claude/ # Claude Code configuration and commands
|
||||
cmd/relspec/ CLI commands (convert, diff, merge, split, edit, inspect, job, ...)
|
||||
pkg/models/ Core model: Database > Schema > Table > Column/Constraint/Index/Relationship
|
||||
pkg/readers/<fmt> One reader per format
|
||||
pkg/writers/<fmt> One writer per format
|
||||
pkg/diff, merge, inspector, jobs, transform, ui, pgsql, sqltypes ...
|
||||
examples/ Sample files
|
||||
tests/ Integration tests and assets
|
||||
docs/ Feature docs
|
||||
```
|
||||
|
||||
## Adding New Readers
|
||||
## Adding a Reader
|
||||
|
||||
To add a new input format reader:
|
||||
1. Create `pkg/readers/<format>/reader.go` with `NewReader(options *readers.ReaderOptions)`.
|
||||
2. Implement `readers.Reader`: `ReadDatabase`, `ReadSchema`, `ReadTable`.
|
||||
3. Add `reader_test.go` in the same package.
|
||||
4. Register the format in the CLI switches (`cmd/relspec/convert.go`, `diff.go`, `edit.go`, ...).
|
||||
5. Add a `README.md` in the reader directory.
|
||||
|
||||
1. Create a new file in `pkg/readers/` (e.g., `myformat_reader.go`)
|
||||
2. Implement the `Reader` interface:
|
||||
```go
|
||||
type Reader interface {
|
||||
Read(source string) (*models.Schema, error)
|
||||
}
|
||||
```
|
||||
3. Add tests in `pkg/readers/myformat_reader_test.go`
|
||||
4. Register the reader in the CLI
|
||||
## Adding a Writer
|
||||
|
||||
## Adding New Writers
|
||||
1. Create `pkg/writers/<format>/writer.go` with `NewWriter(options *writers.WriterOptions)`.
|
||||
2. Implement `writers.Writer`: `WriteDatabase`, `WriteSchema`, `WriteTable`.
|
||||
3. Add `writer_test.go` in the same package.
|
||||
4. Register the format in the CLI switches.
|
||||
5. Add a `README.md` in the writer directory.
|
||||
|
||||
To add a new output format writer:
|
||||
## Code Rules
|
||||
|
||||
1. Create a new file in `pkg/writers/` (e.g., `myformat_writer.go`)
|
||||
2. Implement the `Writer` interface:
|
||||
```go
|
||||
type Writer interface {
|
||||
Write(schema *models.Schema, destination string) error
|
||||
}
|
||||
```
|
||||
3. Add tests in `pkg/writers/myformat_writer_test.go`
|
||||
4. Register the writer in the CLI
|
||||
|
||||
## Code Style
|
||||
|
||||
- Follow standard Go conventions
|
||||
- Use `gofmt` for formatting
|
||||
- Run `go vet` to check for issues
|
||||
- Use meaningful variable and function names
|
||||
- Add comments for exported functions and types
|
||||
- Format with gofumpt/goimports (`make fmt`); `make check` must pass.
|
||||
- Iterate `Table.Columns`, `Constraints`, `Indexes`, `Relationships` in sorted order (maps are unordered; output must be deterministic).
|
||||
- Every writer stamps `buildinfo.GeneratedComment()` in its file header.
|
||||
- Comment exported functions and types.
|
||||
|
||||
## Testing
|
||||
|
||||
- Write unit tests for all new functionality
|
||||
- Aim for >80% code coverage
|
||||
- Use table-driven tests where appropriate
|
||||
- Include both positive and negative test cases
|
||||
- Tests live in the same package as the code.
|
||||
- Table-driven tests; cover positive and negative cases.
|
||||
- Reuse existing test data in `tests/` and `examples/` before adding new data.
|
||||
- Tests run with `-race`.
|
||||
|
||||
### Running Tests
|
||||
## Commits
|
||||
|
||||
```bash
|
||||
# All tests
|
||||
go test ./...
|
||||
- Format: `type(scope): description`
|
||||
- Types: `feat`, `fix`, `docs`, `test`, `refactor`, `chore`, `ci`
|
||||
- Keep commits focused. Reference issues where applicable.
|
||||
|
||||
# With coverage
|
||||
go test -cover ./...
|
||||
## Pull Requests
|
||||
|
||||
# Verbose output
|
||||
go test -v ./...
|
||||
1. Branch from `master`.
|
||||
2. Add tests; `make test` and `make check` pass.
|
||||
3. Update docs/README if behaviour changes.
|
||||
4. Open a PR with a clear description.
|
||||
|
||||
# Specific package
|
||||
go test ./pkg/readers/...
|
||||
```
|
||||
## Security
|
||||
|
||||
## Committing Changes
|
||||
|
||||
- Write clear, descriptive commit messages
|
||||
- Follow conventional commits format: `type(scope): description`
|
||||
- Types: feat, fix, docs, test, refactor, chore
|
||||
- Example: `feat(readers): add PostgreSQL support`
|
||||
- Keep commits focused and atomic
|
||||
- Reference issues in commit messages when applicable
|
||||
|
||||
## Pull Request Process
|
||||
|
||||
1. Create a feature branch from `master`
|
||||
2. Make your changes
|
||||
3. Add tests for new functionality
|
||||
4. Ensure all tests pass
|
||||
5. Update documentation if needed
|
||||
6. Submit a pull request with a clear description
|
||||
|
||||
## Claude Code Commands
|
||||
|
||||
This project includes Claude Code slash commands for common tasks:
|
||||
|
||||
- `/test` - Run all tests
|
||||
- `/build` - Build the binary
|
||||
- `/lint` - Run linters
|
||||
- `/coverage` - Generate coverage report
|
||||
|
||||
## Questions or Issues?
|
||||
|
||||
- Open an issue for bugs or feature requests
|
||||
- Start a discussion for questions or ideas
|
||||
- Check existing issues before creating new ones
|
||||
Do not report vulnerabilities in public issues. See [SECURITY.md](SECURITY.md).
|
||||
|
||||
## License
|
||||
|
||||
By contributing to RelSpec, you agree that your contributions will be licensed under the Apache License 2.0.
|
||||
Contributions are licensed under the Apache License 2.0.
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
.PHONY: all build test test-unit test-integration lint coverage clean install help docker-up docker-down docker-test docker-test-integration start stop release release-version godoc
|
||||
.PHONY: all build test test-unit test-integration lint coverage clean install help docker-up docker-down docker-test docker-test-integration start stop release release-version rerelease godoc vet fmt fmt-check staticcheck govulncheck check
|
||||
|
||||
# Binary name
|
||||
BINARY_NAME=relspec
|
||||
@@ -14,10 +14,17 @@ GOGET=$(GOCMD) get
|
||||
GOMOD=$(GOCMD) mod
|
||||
GOCLEAN=$(GOCMD) clean
|
||||
|
||||
# Tool versions (compiled on demand via `go run` so they match the local toolchain)
|
||||
GOLANGCI_LINT = go run github.com/golangci/golangci-lint/v2/cmd/golangci-lint@latest
|
||||
STATICCHECK = go run honnef.co/go/tools/cmd/staticcheck@latest
|
||||
GOVULNCHECK = go run golang.org/x/vuln/cmd/govulncheck@latest
|
||||
|
||||
# Version information
|
||||
VERSION := $(shell git describe --tags --always --dirty 2>/dev/null || echo "dev")
|
||||
# VERSION file is the single source of truth (bumped by `make release-version`).
|
||||
# Falls back to git describe for ad-hoc builds where VERSION hasn't been committed yet.
|
||||
VERSION := $(shell [ -f VERSION ] && echo "v$$(cat VERSION | tr -d '[:space:]')" || git describe --tags --always --dirty 2>/dev/null || echo "dev")
|
||||
BUILD_DATE := $(shell date -u +"%Y-%m-%d %H:%M:%S UTC")
|
||||
LDFLAGS := -X 'main.version=$(VERSION)' -X 'main.buildDate=$(BUILD_DATE)'
|
||||
LDFLAGS := -X 'git.warky.dev/wdevs/relspecgo/pkg/buildinfo.Version=$(VERSION)' -X 'git.warky.dev/wdevs/relspecgo/pkg/buildinfo.BuildDate=$(BUILD_DATE)'
|
||||
|
||||
# Auto-detect container runtime (Docker or Podman)
|
||||
CONTAINER_RUNTIME := $(shell \
|
||||
@@ -41,6 +48,29 @@ COMPOSE_CMD := $(shell \
|
||||
|
||||
all: lint test build ## Run linting, tests, and build
|
||||
|
||||
check: vet fmt-check staticcheck govulncheck ## Run vet, gofumpt check, staticcheck, and govulncheck
|
||||
|
||||
vet: ## Run go vet
|
||||
@echo "Running go vet..."
|
||||
$(GOCMD) vet ./...
|
||||
|
||||
fmt: ## Format code (gofumpt + goimports via golangci-lint)
|
||||
@echo "Formatting..."
|
||||
$(GOLANGCI_LINT) fmt --config=.golangci.json
|
||||
|
||||
fmt-check: ## Check formatting (gofumpt + goimports via golangci-lint)
|
||||
@echo "Checking formatting..."
|
||||
@diff=$$($(GOLANGCI_LINT) fmt --diff --config=.golangci.json); \
|
||||
if [ -n "$$diff" ]; then echo "$$diff"; echo "Run: make fmt"; exit 1; fi
|
||||
|
||||
staticcheck: ## Run staticcheck
|
||||
@echo "Running staticcheck..."
|
||||
$(STATICCHECK) ./...
|
||||
|
||||
govulncheck: ## Run govulncheck
|
||||
@echo "Running govulncheck..."
|
||||
$(GOVULNCHECK) ./...
|
||||
|
||||
build: deps ## Build the binary
|
||||
@echo "Building $(BINARY_NAME) $(VERSION)..."
|
||||
@mkdir -p $(BUILD_DIR)
|
||||
@@ -179,7 +209,7 @@ docker-test-integration: docker-up ## Start DB and run integration tests
|
||||
$(GOTEST) -v ./pkg/readers/pgsql/ -count=1 || (make docker-down && exit 1)
|
||||
@make docker-down
|
||||
|
||||
release: ## Create and push a new release tag (auto-increments patch version)
|
||||
release: lint fmt-check test build ## Run lint, format check, tests, build, then create and push a new release tag
|
||||
@echo "Creating new release..."
|
||||
@latest_tag=$$(git describe --tags --abbrev=0 2>/dev/null || echo ""); \
|
||||
if [ -z "$$latest_tag" ]; then \
|
||||
@@ -195,6 +225,15 @@ release: ## Create and push a new release tag (auto-increments patch version)
|
||||
echo "Creating new release: $$version"; \
|
||||
commit_logs=$$(git log "$${latest_tag}..HEAD" --pretty=format:"- %s" --no-merges); \
|
||||
fi; \
|
||||
PKGVER=$${version#v}; \
|
||||
echo "Updating package metadata to $$PKGVER..."; \
|
||||
echo "$$PKGVER" > VERSION; \
|
||||
sed -i "s/^pkgver=.*/pkgver=$$PKGVER/" linux/arch/PKGBUILD; \
|
||||
sed -i "s/^Version:.*/Version: $$PKGVER/" linux/centos/relspec.spec; \
|
||||
git add VERSION linux/arch/PKGBUILD linux/centos/relspec.spec; \
|
||||
if ! git diff --cached --quiet; then \
|
||||
git commit -m "chore(release): update package version to $$PKGVER"; \
|
||||
fi; \
|
||||
if [ -z "$$commit_logs" ]; then \
|
||||
tag_message="Release $$version"; \
|
||||
else \
|
||||
@@ -204,21 +243,30 @@ release: ## Create and push a new release tag (auto-increments patch version)
|
||||
git push origin "$$version"; \
|
||||
echo "Tag $$version created and pushed to remote repository."
|
||||
|
||||
release-version: ## Auto-increment patch version, update package files, commit, tag, and push
|
||||
@CURRENT=$$(git describe --tags --abbrev=0 2>/dev/null || echo "v0.0.0"); \
|
||||
MAJOR=$$(echo $$CURRENT | sed 's/v\([0-9]*\)\.\([0-9]*\)\.\([0-9]*\).*/\1/'); \
|
||||
MINOR=$$(echo $$CURRENT | sed 's/v\([0-9]*\)\.\([0-9]*\)\.\([0-9]*\).*/\2/'); \
|
||||
PATCH=$$(echo $$CURRENT | sed 's/v\([0-9]*\)\.\([0-9]*\)\.\([0-9]*\).*/\3/'); \
|
||||
NEXT="v$$MAJOR.$$MINOR.$$((PATCH + 1))"; \
|
||||
release-version: lint fmt-check ## Run lint and format check, then auto-increment patch version, update VERSION/package files, commit, tag, and push
|
||||
@CURRENT=$$([ -f VERSION ] && cat VERSION | tr -d '[:space:]' || (git describe --tags --abbrev=0 2>/dev/null | sed 's/^v//') || echo "0.0.0"); \
|
||||
MAJOR=$$(echo $$CURRENT | cut -d. -f1); \
|
||||
MINOR=$$(echo $$CURRENT | cut -d. -f2); \
|
||||
PATCH=$$(echo $$CURRENT | cut -d. -f3); \
|
||||
PKGVER="$$MAJOR.$$MINOR.$$((PATCH + 1))"; \
|
||||
echo "Current: $$CURRENT → Next: $$NEXT"; \
|
||||
NEXT="v$$PKGVER"; \
|
||||
echo "Current: v$$CURRENT → Next: $$NEXT"; \
|
||||
echo "$$PKGVER" > VERSION; \
|
||||
sed -i "s/^pkgver=.*/pkgver=$$PKGVER/" linux/arch/PKGBUILD; \
|
||||
sed -i "s/^Version:.*/Version: $$PKGVER/" linux/centos/relspec.spec; \
|
||||
git add linux/arch/PKGBUILD linux/centos/relspec.spec; \
|
||||
git add VERSION linux/arch/PKGBUILD linux/centos/relspec.spec; \
|
||||
git commit -m "chore(release): update package version to $$PKGVER"; \
|
||||
git tag -a "$$NEXT" -m "Release $$NEXT"; \
|
||||
git push origin HEAD "$$NEXT"; \
|
||||
echo "Pushed $$NEXT — release workflow triggered"
|
||||
|
||||
rerelease: lint fmt-check ## Move the latest tag to HEAD and force push it
|
||||
@TAG=$$(git describe --tags --abbrev=0 2>/dev/null); \
|
||||
if [ -z "$$TAG" ]; then echo "No existing tags found"; exit 1; fi; \
|
||||
echo "Moving $$TAG to $$(git rev-parse --short HEAD)"; \
|
||||
git tag -f -a "$$TAG" -m "Release $$TAG" HEAD; \
|
||||
git push --force origin "$$TAG"; \
|
||||
echo "Pushed $$TAG — release workflow triggered"
|
||||
|
||||
help: ## Display this help screen
|
||||
@grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-20s\033[0m %s\n", $$1, $$2}'
|
||||
|
||||
@@ -23,6 +23,8 @@ go install -v git.warky.dev/wdevs/relspecgo/cmd/relspec@latest
|
||||
| **Readers** | `bun` `dbml` `dctx` `drawdb` `drizzle` `gorm` `graphql` `json` `mssql` `pgsql` `prisma` `sqldir` `sqlite` `typeorm` `yaml` |
|
||||
| **Writers** | `bun` `dbml` `dctx` `drawdb` `drizzle` `gorm` `graphql` `json` `mssql` `pgsql` `prisma` `sqlexec` `sqlite` `template` `typeorm` `yaml` |
|
||||
|
||||
See [docs/FORMAT_EXAMPLES.md](docs/FORMAT_EXAMPLES.md) for usage examples covering every format.
|
||||
|
||||
## Commands
|
||||
|
||||
### `convert` — Schema conversion
|
||||
@@ -40,6 +42,26 @@ relspec convert --from pgsql --from-conn "postgres://..." --to sqlite --to-path
|
||||
|
||||
# Multiple input files merged
|
||||
relspec convert --from json --from-list "a.json,b.json" --to yaml --to-path merged.yaml
|
||||
|
||||
# Watch mode: regenerate whenever the source file(s) change (Ctrl-C to stop)
|
||||
relspec convert --from dbml --from-path schema.dbml --to gorm --to-path models/ --package models --watch
|
||||
```
|
||||
|
||||
`--watch` works with `--from-path` and `--from-list` (not live database
|
||||
connections or `--dry-run`). Source files are polled every `--watch-interval`
|
||||
(default 500ms), a directory source is watched recursively, and the output path
|
||||
is ignored so generating into the source tree does not loop. Conversion errors
|
||||
are printed and watching continues.
|
||||
|
||||
### `batch` — Convert many inputs in one run
|
||||
|
||||
Converts each input independently (one output per input, unlike `--from-list`
|
||||
which merges). `--input` takes paths or globs; outputs go to `--to-dir`.
|
||||
Use `--keep-going` to continue past failures (exit is still non-zero) and
|
||||
`--dry-run` to validate without writing. For named workflows see `relspec job run`.
|
||||
|
||||
```bash
|
||||
relspec batch --from dbml --input "schemas/*.dbml" --to json --to-dir out/
|
||||
```
|
||||
|
||||
PostgreSQL connections opened by relspec set `application_name` by default to
|
||||
@@ -126,7 +148,7 @@ relspec job run build-schema
|
||||
version: 1
|
||||
jobs:
|
||||
build-schema:
|
||||
command: convert # closed allow-list: convert | merge | scripts-list
|
||||
command: convert # closed allow-list: convert | merge | split | scripts-list | scripts-exec | templ | inspect | diff
|
||||
description: Merge the DBML sources and emit PostgreSQL DDL
|
||||
inputs:
|
||||
- path: schema/core.dbml
|
||||
@@ -147,7 +169,9 @@ resolved relative to the job file and may not escape it, and remote database
|
||||
credentials are referenced by environment-variable name (`conn_env:`) and
|
||||
redacted from logs. The whole plan — unknown commands/formats, duplicate job
|
||||
names, missing inputs, path traversal, dependency cycles — is validated before
|
||||
any job runs. See [docs/JOB_FILES.md](docs/JOB_FILES.md).
|
||||
any job runs. Path-like fields also support `${NAME}` environment-variable
|
||||
references, which are expanded and safety-checked during pre-flight. See
|
||||
[docs/JOB_FILES.md](docs/JOB_FILES.md).
|
||||
|
||||
### `edit` — Interactive TUI editor
|
||||
|
||||
@@ -214,6 +238,23 @@ see [`bun`'s `--array-nullable`](./pkg/writers/bun/README.md#nullablearrays)
|
||||
flag for nullable-array handling. The `SqlXxxArray` wrapper types remain
|
||||
available in `pkg/sqltypes` and are still used by the `gorm` writer.
|
||||
|
||||
#### Custom type mapping
|
||||
|
||||
Override the built-in SQL → Go mapping of the `bun` and `gorm` writers with the
|
||||
repeatable `--type-map sqltype=gotype` flag:
|
||||
|
||||
```bash
|
||||
relspec convert --from pgsql --from-conn "$DSN" --to gorm --to-path models.go \
|
||||
--type-map uuid=string --type-map jsonb=json.RawMessage
|
||||
```
|
||||
|
||||
SQL type names are matched case-insensitively on the base type (modifiers such
|
||||
as `(10,2)` are ignored; aliases like `int4` resolve to `integer`). NOT NULL
|
||||
columns use the Go type verbatim, nullable columns get a `*` prefix (unless the
|
||||
type is already a pointer, slice, map or `any`), and arrays become `[]gotype`.
|
||||
Unmapped types keep their defaults. The flag does not add imports: use types
|
||||
that need none, or add the import afterwards (e.g. with `goimports`).
|
||||
|
||||
## Contributing
|
||||
|
||||
1. Register or sign in with GitHub at [git.warky.dev](https://git.warky.dev)
|
||||
|
||||
+12
@@ -0,0 +1,12 @@
|
||||
# Security Policy
|
||||
|
||||
## Supported versions
|
||||
Security fixes are released for the latest minor version of RelSpec.
|
||||
|
||||
## Reporting a vulnerability
|
||||
Please do not open a public issue for security problems.
|
||||
|
||||
Report privately by email: warkydevs@gmail.com
|
||||
|
||||
Reports are reviewed on a best-effort basis. No response time or fix timeline is
|
||||
guaranteed. Reporters may be credited in the release notes if they wish.
|
||||
@@ -4,20 +4,25 @@
|
||||
|
||||
- [✔️] **Database Inspector**
|
||||
- [✔️] PostgreSQL driver (reader + writer)
|
||||
- [ ] MySQL driver
|
||||
- [ ] MySQL driver (only MariaDB datatype conversion in pkg/mariadb, no reader/writer)
|
||||
- [✔️] SQLite driver (reader + writer with automatic schema flattening)
|
||||
- [ ] MSSQL driver
|
||||
- [✔️] MSSQL driver (reader + writer, generated and identity columns)
|
||||
- [✔️] Foreign key detection
|
||||
- [✔️] Index extraction
|
||||
- [✔️] .sql file generation (PostgreSQL, SQLite)
|
||||
- [✔️] .dbml: Database Markup Language (DBML) for textual schema representation.
|
||||
- [✔️] Prisma schema support (PSL format) .prisma
|
||||
- [ ] Sequelize (Typescript/Javascript) (Use templates, 💲 Someone can do this, not me)
|
||||
- [✔️] Drizzle ORM support .ts (TypeScript / JavaScript) (Mr. Edd wanted to move from Prisma to Drizzle. If you are bugs, you are welcome to do pull requests or issues)
|
||||
- [☠️] Entity Framework (.NET) model .edmx (Fuck no, EDMX files were bloated, verbose XML nightmares—hard to merge, error-prone, and a pain in teams. Microsoft wisely ditched them in EF Core for code-first. Classic overkill from old MS era.)
|
||||
- [✔️] TypeORM support
|
||||
- [] .hbm.xml / schema.xml: Hibernate/Propel mappings (Java/PHP) (💲 Someone can do this, not me)
|
||||
- [ ] Django models.py (Python classes), Sequelize migrations (JS) (💲 Someone can do this, not me)
|
||||
- [] .avsc: Avro schema (JSON format for data serialization) (💲 Someone can do this, not me)
|
||||
- [ ] .hbm.xml / schema.xml: Hibernate/Propel mappings (Java/PHP) (Use templates, 💲 Someone can do this, not me)
|
||||
- [ ] Django models.py (Python classes), Sequelize migrations (JS) (Use templates, 💲 Someone can do this, not me)
|
||||
- [ ] SQLAlchemy, Python (Use templates, 💲 Someone can do this, not me)
|
||||
- [ ] .avsc: Avro schema (JSON format for data serialization) (Use templates, 💲 Someone can do this, not me)
|
||||
- [ ] Rails schema in db/schema.rb (Use templates, 💲 Someone can do this, not me)
|
||||
- [ ] Laravel migration (Use templates, 💲 Someone can do this, not me)
|
||||
- [ ] Hibernate / JPA (Use templates, 💲 Someone can do this, not me)
|
||||
- [✔️] GraphQL schema generation
|
||||
|
||||
## UI
|
||||
@@ -39,13 +44,19 @@
|
||||
|
||||
## Advanced Features
|
||||
|
||||
- [ ] Dry-run mode for validation
|
||||
- [x] Diff tool for comparing specifications
|
||||
- [ ] Migration script generation
|
||||
- [ ] Dry-run mode for validation (only `job run --dry-run`; not on convert/merge/split)
|
||||
- [✔️] Diff tool for comparing specifications
|
||||
- [✔️] Migration script generation (PostgreSQL diff, live database by default for direct output)
|
||||
- [ ] Custom type mapping configuration
|
||||
- [ ] Batch processing support
|
||||
- [ ] Batch processing support (partial: job files run named workflows via `relspec job run`)
|
||||
- [ ] Watch mode for auto-regeneration
|
||||
|
||||
## Distribution
|
||||
|
||||
- [ ] NSIS Windows installer
|
||||
- [ ] Check https://git.warky.dev/wdevs/relspecgo/releases for new releases and prompt to update
|
||||
- [ ] CI pipeline to build and publish the Windows installer
|
||||
|
||||
## Future Considerations
|
||||
|
||||
- [ ] Web UI for visual editing
|
||||
|
||||
@@ -0,0 +1,214 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
|
||||
var (
|
||||
batchSourceType string
|
||||
batchInputs []string
|
||||
batchTargetType string
|
||||
batchTargetDir string
|
||||
batchPackageName string
|
||||
batchSchemaFilter string
|
||||
batchFlattenSchema bool
|
||||
batchNullableTypes string
|
||||
batchNullableArrays string
|
||||
batchContinueOnError bool
|
||||
batchKeepGoing bool
|
||||
batchDryRun bool
|
||||
)
|
||||
|
||||
var batchCmd = &cobra.Command{
|
||||
Use: "batch",
|
||||
Short: "Convert many input files to a target format in one run",
|
||||
Long: `Convert each input file independently to the target format.
|
||||
|
||||
Unlike 'convert --from-list', which merges all inputs into one output, batch
|
||||
mode writes one output per input into --to-dir. The output is named after the
|
||||
input file (without its extension). Directory-style targets (gorm, bun,
|
||||
drizzle) get a sub-directory per input.
|
||||
|
||||
Inputs are given with --input, which accepts file paths and glob patterns and
|
||||
may be repeated or comma-separated. Inputs are processed in sorted order and
|
||||
duplicates are removed. The command exits non-zero if any input fails.
|
||||
|
||||
For named, multi-step workflows use 'relspec job run' instead.
|
||||
|
||||
Examples:
|
||||
# Convert every DBML file in a directory to JSON
|
||||
relspec batch --from dbml --input "schemas/*.dbml" --to json --to-dir out/
|
||||
|
||||
# Convert specific files to GORM models, one package directory per input
|
||||
relspec batch --from json --input a.json,b.json \
|
||||
--to gorm --to-dir models/ --package models
|
||||
|
||||
# Validate everything first, writing nothing
|
||||
relspec batch --from yaml --input "specs/*.yaml" --to pgsql --to-dir sql/ --dry-run
|
||||
|
||||
# Report all failures instead of stopping at the first
|
||||
relspec batch --from json --input "*.json" --to yaml --to-dir out/ --keep-going`,
|
||||
RunE: runBatch,
|
||||
}
|
||||
|
||||
func init() {
|
||||
batchCmd.Flags().StringVar(&batchSourceType, "from", "", "Source format for every input (dbml, dctx, drawdb, graphql, json, yaml, gorm, bun, drizzle, prisma, typeorm, sqlite)")
|
||||
batchCmd.Flags().StringSliceVar(&batchInputs, "input", nil, "Input file path or glob pattern (repeatable, comma-separated)")
|
||||
batchCmd.Flags().StringVar(&batchTargetType, "to", "", "Target format")
|
||||
batchCmd.Flags().StringVar(&batchTargetDir, "to-dir", "", "Output directory; one output per input is written here")
|
||||
batchCmd.Flags().StringVar(&batchPackageName, "package", "", "Package name (for code generation formats like gorm/bun)")
|
||||
batchCmd.Flags().StringVar(&batchSchemaFilter, "schema", "", "Filter to a specific schema by name")
|
||||
batchCmd.Flags().BoolVar(&batchFlattenSchema, "flatten-schema", false, "Flatten schema.table names to schema_table")
|
||||
batchCmd.Flags().StringVar(&batchNullableTypes, "types", "", "Nullable type package for code-gen writers (bun/gorm)")
|
||||
batchCmd.Flags().StringVar(&batchNullableArrays, "array-nullable", "", "Nullable array representation for the Bun writer")
|
||||
batchCmd.Flags().BoolVar(&batchContinueOnError, "continue-on-error", false, "Prepend \\set ON_ERROR_STOP off to generated SQL (pgsql output only)")
|
||||
batchCmd.Flags().BoolVar(&batchKeepGoing, "keep-going", false, "Process remaining inputs after a failure; still exits non-zero")
|
||||
batchCmd.Flags().BoolVar(&batchDryRun, "dry-run", false, "Read and validate every input and print the plan without writing any output")
|
||||
|
||||
for _, f := range []string{"from", "input", "to", "to-dir"} {
|
||||
if err := batchCmd.MarkFlagRequired(f); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "Error marking %s flag as required: %v\n", f, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// batchDirTargets are writers that emit a directory of files rather than one file.
|
||||
var batchDirTargets = map[string]bool{"gorm": true, "bun": true, "drizzle": true}
|
||||
|
||||
// batchExtensions maps single-file target formats to their output extension.
|
||||
var batchExtensions = map[string]string{
|
||||
"dbml": ".dbml", "dctx": ".dctx", "drawdb": ".ddb", "json": ".json",
|
||||
"yaml": ".yaml", "yml": ".yaml", "pgsql": ".sql", "postgres": ".sql",
|
||||
"postgresql": ".sql", "sql": ".sql", "mssql": ".sql", "sqlserver": ".sql",
|
||||
"mssql2016": ".sql", "mssql2017": ".sql", "mssql2019": ".sql", "mssql2022": ".sql",
|
||||
"sqlite": ".sql", "sqlite3": ".sql", "prisma": ".prisma", "typeorm": ".ts",
|
||||
"graphql": ".graphql", "gql": ".graphql",
|
||||
}
|
||||
|
||||
// expandBatchInputs resolves paths and glob patterns into a sorted,
|
||||
// de-duplicated file list. A pattern that matches nothing is an error.
|
||||
func expandBatchInputs(patterns []string) ([]string, error) {
|
||||
seen := map[string]bool{}
|
||||
var files []string
|
||||
for _, p := range patterns {
|
||||
p = strings.TrimSpace(p)
|
||||
if p == "" {
|
||||
continue
|
||||
}
|
||||
matches, err := filepath.Glob(p)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("invalid pattern %q: %w", p, err)
|
||||
}
|
||||
if len(matches) == 0 {
|
||||
return nil, fmt.Errorf("no files match %q", p)
|
||||
}
|
||||
for _, m := range matches {
|
||||
if !seen[m] {
|
||||
seen[m] = true
|
||||
files = append(files, m)
|
||||
}
|
||||
}
|
||||
}
|
||||
if len(files) == 0 {
|
||||
return nil, fmt.Errorf("no input files given")
|
||||
}
|
||||
sort.Strings(files)
|
||||
return files, nil
|
||||
}
|
||||
|
||||
// batchOutputPaths returns the output path for each input. It errors when two
|
||||
// inputs would collide on the same output name.
|
||||
func batchOutputPaths(files []string, targetType, dir string) ([]string, error) {
|
||||
key := strings.ToLower(targetType)
|
||||
ext := ""
|
||||
if !batchDirTargets[key] {
|
||||
var ok bool
|
||||
if ext, ok = batchExtensions[key]; !ok {
|
||||
return nil, fmt.Errorf("unsupported target format: %s", targetType)
|
||||
}
|
||||
}
|
||||
outs := make([]string, len(files))
|
||||
owner := map[string]string{}
|
||||
for i, f := range files {
|
||||
stem := strings.TrimSuffix(filepath.Base(f), filepath.Ext(f))
|
||||
out := filepath.Join(dir, stem+ext)
|
||||
if prev, dup := owner[out]; dup {
|
||||
return nil, fmt.Errorf("inputs %s and %s would both write %s", prev, f, out)
|
||||
}
|
||||
owner[out] = f
|
||||
outs[i] = out
|
||||
}
|
||||
return outs, nil
|
||||
}
|
||||
|
||||
func runBatch(cmd *cobra.Command, args []string) error {
|
||||
files, err := expandBatchInputs(batchInputs)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
outs, err := batchOutputPaths(files, batchTargetType, batchTargetDir)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
fmt.Fprintf(os.Stderr, "\n=== RelSpec Batch Converter ===\n")
|
||||
fmt.Fprintf(os.Stderr, "Started at: %s\n", getCurrentTimestamp())
|
||||
fmt.Fprintf(os.Stderr, "Inputs: %d file(s), %s -> %s\n\n", len(files), batchSourceType, batchTargetType)
|
||||
|
||||
out := outWriter(cmd)
|
||||
if batchDryRun {
|
||||
fmt.Fprintf(out, "RelSpec batch plan (dry run - nothing written):\n")
|
||||
}
|
||||
|
||||
var failed []string
|
||||
for i, f := range files {
|
||||
fmt.Fprintf(os.Stderr, "[%d/%d] %s -> %s\n", i+1, len(files), f, outs[i])
|
||||
if err := processBatchItem(cmd, f, outs[i]); err != nil {
|
||||
fmt.Fprintf(os.Stderr, " ✗ %v\n", err)
|
||||
failed = append(failed, fmt.Sprintf("%s: %v", f, err))
|
||||
if !batchKeepGoing {
|
||||
break
|
||||
}
|
||||
continue
|
||||
}
|
||||
fmt.Fprintf(os.Stderr, " ✓ done\n")
|
||||
}
|
||||
|
||||
fmt.Fprintf(os.Stderr, "\n=== Batch Complete: %d ok, %d failed ===\n", len(files)-len(failed), len(failed))
|
||||
if len(failed) > 0 {
|
||||
return fmt.Errorf("batch finished with %d failure(s):\n %s", len(failed), strings.Join(failed, "\n "))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func processBatchItem(cmd *cobra.Command, in, outPath string) error {
|
||||
db, err := readDatabaseForConvert(batchSourceType, in, "")
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to read source: %w", err)
|
||||
}
|
||||
finalizeCommentedRefs(db, stderrWarn)
|
||||
|
||||
if batchDryRun {
|
||||
if err := validateWriteTarget(db, batchTargetType, batchPackageName, batchSchemaFilter, ""); err != nil {
|
||||
return fmt.Errorf("dry run validation failed: %w", err)
|
||||
}
|
||||
w := outWriter(cmd)
|
||||
fmt.Fprintf(w, " %s -> %s (database '%s')\n", in, outPath, db.Name)
|
||||
printDryRunPlan(w, db)
|
||||
return nil
|
||||
}
|
||||
|
||||
if err := os.MkdirAll(batchTargetDir, 0o755); err != nil {
|
||||
return fmt.Errorf("failed to create output directory: %w", err)
|
||||
}
|
||||
if err := writeDatabase(db, batchTargetType, outPath, batchPackageName, batchSchemaFilter, batchFlattenSchema, batchNullableTypes, batchNullableArrays, batchContinueOnError, ""); err != nil {
|
||||
return fmt.Errorf("failed to write target: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,143 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func saveBatchState(t *testing.T) {
|
||||
t.Helper()
|
||||
a, b, c, d, e, f, g := batchSourceType, batchInputs, batchTargetType, batchTargetDir, batchPackageName, batchKeepGoing, batchDryRun
|
||||
t.Cleanup(func() {
|
||||
batchSourceType, batchInputs, batchTargetType, batchTargetDir, batchPackageName, batchKeepGoing, batchDryRun = a, b, c, d, e, f, g
|
||||
})
|
||||
}
|
||||
|
||||
func TestRunBatch_ConvertsEachInput(t *testing.T) {
|
||||
saveBatchState(t)
|
||||
dir := t.TempDir()
|
||||
writeTestJSON(t, filepath.Join(dir, "a.json"), []string{"users"})
|
||||
writeTestJSON(t, filepath.Join(dir, "b.json"), []string{"posts"})
|
||||
outDir := filepath.Join(dir, "out")
|
||||
|
||||
batchSourceType, batchTargetType, batchTargetDir = "json", "yaml", outDir
|
||||
batchPackageName, batchKeepGoing, batchDryRun = "", false, false
|
||||
batchInputs = []string{filepath.Join(dir, "*.json")}
|
||||
|
||||
cmd, _ := newDryRunCmd()
|
||||
if err := runBatch(cmd, nil); err != nil {
|
||||
t.Fatalf("batch: %v", err)
|
||||
}
|
||||
for _, name := range []string{"a.yaml", "b.yaml"} {
|
||||
if _, err := os.Stat(filepath.Join(outDir, name)); err != nil {
|
||||
t.Errorf("expected %s: %v", name, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunBatch_DryRunWritesNothing(t *testing.T) {
|
||||
saveBatchState(t)
|
||||
dir := t.TempDir()
|
||||
writeTestJSON(t, filepath.Join(dir, "a.json"), []string{"users"})
|
||||
outDir := filepath.Join(dir, "out")
|
||||
|
||||
batchSourceType, batchTargetType, batchTargetDir = "json", "yaml", outDir
|
||||
batchPackageName, batchKeepGoing, batchDryRun = "", false, true
|
||||
batchInputs = []string{filepath.Join(dir, "a.json")}
|
||||
|
||||
cmd, buf := newDryRunCmd()
|
||||
if err := runBatch(cmd, nil); err != nil {
|
||||
t.Fatalf("dry run: %v", err)
|
||||
}
|
||||
if _, err := os.Stat(outDir); !os.IsNotExist(err) {
|
||||
t.Fatal("dry run must not create the output directory")
|
||||
}
|
||||
if !strings.Contains(buf.String(), "users") || !strings.Contains(buf.String(), "a.yaml") {
|
||||
t.Errorf("plan incomplete:\n%s", buf.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunBatch_FailureHandling(t *testing.T) {
|
||||
saveBatchState(t)
|
||||
dir := t.TempDir()
|
||||
writeTestJSON(t, filepath.Join(dir, "a.json"), []string{"users"})
|
||||
if err := os.WriteFile(filepath.Join(dir, "b.json"), []byte("{not json"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
writeTestJSON(t, filepath.Join(dir, "c.json"), []string{"posts"})
|
||||
outDir := filepath.Join(dir, "out")
|
||||
|
||||
batchSourceType, batchTargetType, batchTargetDir = "json", "yaml", outDir
|
||||
batchPackageName, batchDryRun = "", false
|
||||
batchInputs = []string{filepath.Join(dir, "*.json")}
|
||||
cmd, _ := newDryRunCmd()
|
||||
|
||||
// Default: stop at first failure.
|
||||
batchKeepGoing = false
|
||||
err := runBatch(cmd, nil)
|
||||
if err == nil || !strings.Contains(err.Error(), "b.json") {
|
||||
t.Fatalf("expected failure naming b.json, got %v", err)
|
||||
}
|
||||
if _, statErr := os.Stat(filepath.Join(outDir, "c.yaml")); !os.IsNotExist(statErr) {
|
||||
t.Error("c.json should not be processed without --keep-going")
|
||||
}
|
||||
|
||||
// --keep-going: remaining inputs are processed, exit still fails.
|
||||
batchKeepGoing = true
|
||||
if err := runBatch(cmd, nil); err == nil {
|
||||
t.Fatal("expected non-zero result with --keep-going")
|
||||
}
|
||||
if _, statErr := os.Stat(filepath.Join(outDir, "c.yaml")); statErr != nil {
|
||||
t.Errorf("c.yaml should be written with --keep-going: %v", statErr)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExpandBatchInputs(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
for _, n := range []string{"b.json", "a.json"} {
|
||||
if err := os.WriteFile(filepath.Join(dir, n), []byte("{}"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
got, err := expandBatchInputs([]string{filepath.Join(dir, "*.json"), filepath.Join(dir, "a.json")})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(got) != 2 || filepath.Base(got[0]) != "a.json" || filepath.Base(got[1]) != "b.json" {
|
||||
t.Errorf("want sorted deduped [a b], got %v", got)
|
||||
}
|
||||
if _, err := expandBatchInputs([]string{filepath.Join(dir, "*.nope")}); err == nil {
|
||||
t.Error("unmatched pattern should error")
|
||||
}
|
||||
}
|
||||
|
||||
func TestBatchOutputPaths(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
files []string
|
||||
target string
|
||||
want []string
|
||||
wantErr string
|
||||
}{
|
||||
{"file target", []string{"x/a.dbml"}, "json", []string{"out/a.json"}, ""},
|
||||
{"dir target", []string{"x/a.json"}, "gorm", []string{"out/a"}, ""},
|
||||
{"collision", []string{"x/a.json", "y/a.json"}, "yaml", nil, "both write"},
|
||||
{"unsupported", []string{"a.json"}, "nope", nil, "unsupported target"},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got, err := batchOutputPaths(tt.files, tt.target, "out")
|
||||
if tt.wantErr != "" {
|
||||
if err == nil || !strings.Contains(err.Error(), tt.wantErr) {
|
||||
t.Fatalf("want error %q, got %v", tt.wantErr, err)
|
||||
}
|
||||
return
|
||||
}
|
||||
if err != nil || len(got) != len(tt.want) || got[0] != filepath.FromSlash(tt.want[0]) {
|
||||
t.Fatalf("got %v, %v; want %v", got, err, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
+153
-15
@@ -3,6 +3,7 @@ package main
|
||||
import (
|
||||
stdjson "encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"strings"
|
||||
"time"
|
||||
@@ -21,6 +22,7 @@ import (
|
||||
"git.warky.dev/wdevs/relspecgo/pkg/readers/graphql"
|
||||
"git.warky.dev/wdevs/relspecgo/pkg/readers/json"
|
||||
"git.warky.dev/wdevs/relspecgo/pkg/readers/mssql"
|
||||
"git.warky.dev/wdevs/relspecgo/pkg/readers/mysql"
|
||||
"git.warky.dev/wdevs/relspecgo/pkg/readers/pgsql"
|
||||
"git.warky.dev/wdevs/relspecgo/pkg/readers/prisma"
|
||||
"git.warky.dev/wdevs/relspecgo/pkg/readers/sqlite"
|
||||
@@ -36,6 +38,7 @@ import (
|
||||
wgraphql "git.warky.dev/wdevs/relspecgo/pkg/writers/graphql"
|
||||
wjson "git.warky.dev/wdevs/relspecgo/pkg/writers/json"
|
||||
wmssql "git.warky.dev/wdevs/relspecgo/pkg/writers/mssql"
|
||||
wmysql "git.warky.dev/wdevs/relspecgo/pkg/writers/mysql"
|
||||
wpgsql "git.warky.dev/wdevs/relspecgo/pkg/writers/pgsql"
|
||||
wprisma "git.warky.dev/wdevs/relspecgo/pkg/writers/prisma"
|
||||
wsqlite "git.warky.dev/wdevs/relspecgo/pkg/writers/sqlite"
|
||||
@@ -57,6 +60,9 @@ var (
|
||||
convertNullableArrays string
|
||||
convertContinueOnError bool
|
||||
convertExtraFields string
|
||||
convertDryRun bool
|
||||
convertWatch bool
|
||||
convertWatchInterval time.Duration
|
||||
)
|
||||
|
||||
var convertCmd = &cobra.Command{
|
||||
@@ -165,7 +171,11 @@ Examples:
|
||||
|
||||
# Convert SQLite to PostgreSQL SQL
|
||||
relspec convert --from sqlite --from-path database.db \
|
||||
--to pgsql --to-path schema.sql`,
|
||||
--to pgsql --to-path schema.sql
|
||||
|
||||
# Regenerate GORM models every time the DBML file changes
|
||||
relspec convert --from dbml --from-path schema.dbml \
|
||||
--to gorm --to-path models/ --package models --watch`,
|
||||
RunE: runConvert,
|
||||
}
|
||||
|
||||
@@ -185,6 +195,11 @@ func init() {
|
||||
convertCmd.Flags().BoolVar(&convertContinueOnError, "continue-on-error", false, "Prepend \\set ON_ERROR_STOP off to generated SQL so psql continues past errors (pgsql output only)")
|
||||
convertCmd.Flags().StringVar(&convertExtraFields, "extra-fields", "", "Path to JSON file containing extra Bun model fields to inject (bun output only); fields support target_table, name, type, bun_tag, json_tag, comment")
|
||||
|
||||
convertCmd.Flags().BoolVar(&convertDryRun, "dry-run", false, "Read and validate the input and print the plan without writing any output")
|
||||
|
||||
convertCmd.Flags().BoolVar(&convertWatch, "watch", false, "Watch the source files (--from-path or --from-list) and regenerate the output whenever they change")
|
||||
convertCmd.Flags().DurationVar(&convertWatchInterval, "watch-interval", 500*time.Millisecond, "Polling interval used by --watch")
|
||||
|
||||
err := convertCmd.MarkFlagRequired("from")
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "Error marking from flag as required: %v\n", err)
|
||||
@@ -200,6 +215,13 @@ func init() {
|
||||
}
|
||||
|
||||
func runConvert(cmd *cobra.Command, args []string) error {
|
||||
if convertWatch {
|
||||
return runConvertWatch(cmd.Context(), os.Stderr, func() error { return runConvertOnce(cmd) })
|
||||
}
|
||||
return runConvertOnce(cmd)
|
||||
}
|
||||
|
||||
func runConvertOnce(cmd *cobra.Command) error {
|
||||
fmt.Fprintf(os.Stderr, "\n=== RelSpec Schema Converter ===\n")
|
||||
fmt.Fprintf(os.Stderr, "Started at: %s\n\n", getCurrentTimestamp())
|
||||
|
||||
@@ -229,6 +251,7 @@ func runConvert(cmd *cobra.Command, args []string) error {
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to read source: %w", err)
|
||||
}
|
||||
finalizeCommentedRefs(db, stderrWarn)
|
||||
|
||||
fmt.Fprintf(os.Stderr, " ✓ Successfully read database '%s'\n", db.Name)
|
||||
fmt.Fprintf(os.Stderr, " Found: %d schema(s)\n", len(db.Schemas))
|
||||
@@ -239,6 +262,22 @@ func runConvert(cmd *cobra.Command, args []string) error {
|
||||
}
|
||||
fmt.Fprintf(os.Stderr, " Found: %d table(s)\n\n", totalTables)
|
||||
|
||||
if convertDryRun {
|
||||
if err := validateWriteTarget(db, convertTargetType, convertPackageName, convertSchemaFilter, convertExtraFields); err != nil {
|
||||
return fmt.Errorf("dry run validation failed: %w", err)
|
||||
}
|
||||
out := outWriter(cmd)
|
||||
fmt.Fprintf(out, "RelSpec convert plan (dry run - nothing written):\n")
|
||||
fmt.Fprintf(out, " Input: %s database '%s'\n", convertSourceType, db.Name)
|
||||
fmt.Fprintf(out, " Output: %s -> %s\n", convertTargetType, convertTargetPath)
|
||||
if convertSchemaFilter != "" {
|
||||
fmt.Fprintf(out, " Schema filter: %s\n", convertSchemaFilter)
|
||||
}
|
||||
printDryRunPlan(out, db)
|
||||
fmt.Fprintf(os.Stderr, "=== Dry run complete: no output written ===\n\n")
|
||||
return nil
|
||||
}
|
||||
|
||||
// Write to target format
|
||||
fmt.Fprintf(os.Stderr, "[2/2] Writing to target format...\n")
|
||||
fmt.Fprintf(os.Stderr, " Format: %s\n", convertTargetType)
|
||||
@@ -261,6 +300,18 @@ func runConvert(cmd *cobra.Command, args []string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// finalizeCommentedRefs resolves DBML `// Ref:` comments against the fully
|
||||
// loaded model and warns about refs whose target is not loaded.
|
||||
func finalizeCommentedRefs(db *models.Database, warn func(string)) {
|
||||
for _, w := range dbml.ResolveCommentedRefs(db, true) {
|
||||
warn(w)
|
||||
}
|
||||
}
|
||||
|
||||
func stderrWarn(msg string) {
|
||||
fmt.Fprintf(os.Stderr, " ⚠ %s\n", msg)
|
||||
}
|
||||
|
||||
func readDatabaseListForConvert(dbType string, files []string) (*models.Database, error) {
|
||||
if len(files) == 0 {
|
||||
return nil, fmt.Errorf("file list is empty")
|
||||
@@ -367,6 +418,12 @@ func readDatabaseForConvert(dbType, filePath, connString string) (*models.Databa
|
||||
}
|
||||
reader = mssql.NewReader(newReaderOptions("", connString))
|
||||
|
||||
case "mysql", "mariadb":
|
||||
if connString == "" {
|
||||
return nil, fmt.Errorf("connection string is required for MySQL format")
|
||||
}
|
||||
reader = mysql.NewReader(newReaderOptions("", connString))
|
||||
|
||||
case "sqlite", "sqlite3":
|
||||
// SQLite can use either file path or connection string
|
||||
dbPath := filePath
|
||||
@@ -395,23 +452,12 @@ func writeDatabase(db *models.Database, dbType, outputPath, packageName, schemaF
|
||||
|
||||
writerOpts := newWriterOptions(outputPath, packageName, flattenSchema, nullableTypes, nullableArrays, continueOnError)
|
||||
if extraFields != "" {
|
||||
if !strings.EqualFold(dbType, "bun") {
|
||||
return fmt.Errorf("--extra-fields is only supported for Bun output")
|
||||
}
|
||||
extraFieldsJSON, err := os.ReadFile(extraFields)
|
||||
extraFieldsJSON, err := loadExtraFields(dbType, extraFields)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to read --extra-fields file %q: %w", extraFields, err)
|
||||
}
|
||||
|
||||
var parsed []wbun.ExtraFieldConfig
|
||||
if err := stdjson.Unmarshal(extraFieldsJSON, &parsed); err != nil {
|
||||
return fmt.Errorf("invalid --extra-fields JSON in %q: %w", extraFields, err)
|
||||
}
|
||||
if len(parsed) == 0 {
|
||||
return fmt.Errorf("--extra-fields must contain at least one field")
|
||||
return err
|
||||
}
|
||||
writerOpts.Metadata = map[string]interface{}{
|
||||
"extra_fields": string(extraFieldsJSON),
|
||||
"extra_fields": extraFieldsJSON,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -452,6 +498,9 @@ func writeDatabase(db *models.Database, dbType, outputPath, packageName, schemaF
|
||||
case "mssql", "sqlserver", "mssql2016", "mssql2017", "mssql2019", "mssql2022":
|
||||
writer = wmssql.NewWriter(writerOpts)
|
||||
|
||||
case "mysql", "mariadb":
|
||||
writer = wmysql.NewWriter(writerOpts)
|
||||
|
||||
case "sqlite", "sqlite3":
|
||||
writer = wsqlite.NewWriter(writerOpts)
|
||||
|
||||
@@ -516,6 +565,95 @@ func writeDatabase(db *models.Database, dbType, outputPath, packageName, schemaF
|
||||
return nil
|
||||
}
|
||||
|
||||
// loadExtraFields reads and validates the --extra-fields JSON file, returning
|
||||
// its raw content.
|
||||
func loadExtraFields(dbType, path string) (string, error) {
|
||||
if !strings.EqualFold(dbType, "bun") {
|
||||
return "", fmt.Errorf("--extra-fields is only supported for Bun output")
|
||||
}
|
||||
extraFieldsJSON, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to read --extra-fields file %q: %w", path, err)
|
||||
}
|
||||
|
||||
var parsed []wbun.ExtraFieldConfig
|
||||
if err := stdjson.Unmarshal(extraFieldsJSON, &parsed); err != nil {
|
||||
return "", fmt.Errorf("invalid --extra-fields JSON in %q: %w", path, err)
|
||||
}
|
||||
if len(parsed) == 0 {
|
||||
return "", fmt.Errorf("--extra-fields must contain at least one field")
|
||||
}
|
||||
return string(extraFieldsJSON), nil
|
||||
}
|
||||
|
||||
// validateWriteTarget performs the checks writeDatabase makes before writing,
|
||||
// without constructing a writer or touching the output path. Used by --dry-run.
|
||||
func validateWriteTarget(db *models.Database, dbType, packageName, schemaFilter, extraFields string) error {
|
||||
if extraFields != "" {
|
||||
if _, err := loadExtraFields(dbType, extraFields); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
switch strings.ToLower(dbType) {
|
||||
case "dbml", "dctx", "drawdb", "json", "yaml", "yml", "drizzle",
|
||||
"pgsql", "postgres", "postgresql", "sql",
|
||||
"mssql", "sqlserver", "mssql2016", "mssql2017", "mssql2019", "mssql2022",
|
||||
"sqlite", "sqlite3", "prisma", "typeorm", "graphql", "gql":
|
||||
case "gorm", "bun":
|
||||
if packageName == "" {
|
||||
return fmt.Errorf("package name is required for %s format (use --package flag)", dbType)
|
||||
}
|
||||
default:
|
||||
return fmt.Errorf("unsupported target format: %s", dbType)
|
||||
}
|
||||
|
||||
if schemaFilter != "" {
|
||||
for _, schema := range db.Schemas {
|
||||
if schema.Name == schemaFilter {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
return fmt.Errorf("schema '%s' not found in database. Available schemas: %v",
|
||||
schemaFilter, getSchemaNames(db))
|
||||
}
|
||||
|
||||
if strings.EqualFold(dbType, "dctx") {
|
||||
if len(db.Schemas) == 0 {
|
||||
return fmt.Errorf("no schemas found in database")
|
||||
}
|
||||
if len(db.Schemas) > 1 {
|
||||
return fmt.Errorf("multiple schemas found, please specify which schema to export using --schema flag. Available schemas: %v",
|
||||
getSchemaNames(db))
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// outWriter returns the command's stdout, falling back to os.Stdout when the
|
||||
// command is nil (tests call the run functions directly).
|
||||
func outWriter(cmd *cobra.Command) io.Writer {
|
||||
if cmd == nil {
|
||||
return os.Stdout
|
||||
}
|
||||
return cmd.OutOrStdout()
|
||||
}
|
||||
|
||||
// printDryRunPlan prints the schemas and tables that would be written.
|
||||
func printDryRunPlan(out io.Writer, db *models.Database) {
|
||||
for _, schema := range db.Schemas {
|
||||
names := make([]string, 0, len(schema.Tables))
|
||||
for _, t := range schema.Tables {
|
||||
names = append(names, t.Name)
|
||||
}
|
||||
fmt.Fprintf(out, " schema %q: %d table(s)", schema.Name, len(schema.Tables))
|
||||
if len(names) > 0 {
|
||||
fmt.Fprintf(out, " [%s]", strings.Join(names, ", "))
|
||||
}
|
||||
fmt.Fprintln(out)
|
||||
}
|
||||
}
|
||||
|
||||
// getSchemaNames returns a slice of schema names from a database
|
||||
func getSchemaNames(db *models.Database) []string {
|
||||
names := make([]string, len(db.Schemas))
|
||||
|
||||
@@ -46,7 +46,7 @@ func TestReadDatabaseListForConvert_MultipleFiles(t *testing.T) {
|
||||
|
||||
func TestReadDatabaseListForConvert_PathWithSpaces(t *testing.T) {
|
||||
spacedDir := filepath.Join(t.TempDir(), "my schema files")
|
||||
if err := os.MkdirAll(spacedDir, 0755); err != nil {
|
||||
if err := os.MkdirAll(spacedDir, 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
file := filepath.Join(spacedDir, "my users schema.json")
|
||||
@@ -63,7 +63,7 @@ func TestReadDatabaseListForConvert_PathWithSpaces(t *testing.T) {
|
||||
|
||||
func TestReadDatabaseListForConvert_MultipleFilesPathWithSpaces(t *testing.T) {
|
||||
spacedDir := filepath.Join(t.TempDir(), "my schema files")
|
||||
if err := os.MkdirAll(spacedDir, 0755); err != nil {
|
||||
if err := os.MkdirAll(spacedDir, 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
file1 := filepath.Join(spacedDir, "users schema.json")
|
||||
@@ -154,7 +154,7 @@ func TestRunConvert_FromListEndToEndPathWithSpaces(t *testing.T) {
|
||||
defer restoreConvertState(saved)
|
||||
|
||||
spacedDir := filepath.Join(t.TempDir(), "my schema dir")
|
||||
if err := os.MkdirAll(spacedDir, 0755); err != nil {
|
||||
if err := os.MkdirAll(spacedDir, 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
file1 := filepath.Join(spacedDir, "users schema.json")
|
||||
|
||||
+8
-8
@@ -229,43 +229,43 @@ func readDatabase(dbType, filePath, connString, label string) (*models.Database,
|
||||
if filePath == "" {
|
||||
return nil, fmt.Errorf("%s: file path is required for DBML format", label)
|
||||
}
|
||||
reader = dbml.NewReader(&readers.ReaderOptions{FilePath: filePath})
|
||||
reader = dbml.NewReader(newReaderOptions(filePath, ""))
|
||||
|
||||
case "dctx":
|
||||
if filePath == "" {
|
||||
return nil, fmt.Errorf("%s: file path is required for DCTX format", label)
|
||||
}
|
||||
reader = dctx.NewReader(&readers.ReaderOptions{FilePath: filePath})
|
||||
reader = dctx.NewReader(newReaderOptions(filePath, ""))
|
||||
|
||||
case "drawdb":
|
||||
if filePath == "" {
|
||||
return nil, fmt.Errorf("%s: file path is required for DrawDB format", label)
|
||||
}
|
||||
reader = drawdb.NewReader(&readers.ReaderOptions{FilePath: filePath})
|
||||
reader = drawdb.NewReader(newReaderOptions(filePath, ""))
|
||||
|
||||
case "json":
|
||||
if filePath == "" {
|
||||
return nil, fmt.Errorf("%s: file path is required for JSON format", label)
|
||||
}
|
||||
reader = json.NewReader(&readers.ReaderOptions{FilePath: filePath})
|
||||
reader = json.NewReader(newReaderOptions(filePath, ""))
|
||||
|
||||
case "yaml":
|
||||
if filePath == "" {
|
||||
return nil, fmt.Errorf("%s: file path is required for YAML format", label)
|
||||
}
|
||||
reader = yaml.NewReader(&readers.ReaderOptions{FilePath: filePath})
|
||||
reader = yaml.NewReader(newReaderOptions(filePath, ""))
|
||||
|
||||
case "sqldir", "scripts", "scriptdir":
|
||||
if filePath == "" {
|
||||
return nil, fmt.Errorf("%s: file path is required for SQL directory format", label)
|
||||
}
|
||||
reader = sqldir.NewReader(&readers.ReaderOptions{FilePath: filePath})
|
||||
reader = sqldir.NewReader(newReaderOptions(filePath, ""))
|
||||
|
||||
case "pgsql", "postgres", "postgresql":
|
||||
if connString == "" {
|
||||
return nil, fmt.Errorf("%s: connection string is required for PostgreSQL format", label)
|
||||
}
|
||||
reader = pgsql.NewReader(&readers.ReaderOptions{ConnectionString: connString})
|
||||
reader = pgsql.NewReader(newReaderOptions("", connString))
|
||||
|
||||
case "sqlite", "sqlite3":
|
||||
// SQLite can use either file path or connection string
|
||||
@@ -276,7 +276,7 @@ func readDatabase(dbType, filePath, connString, label string) (*models.Database,
|
||||
if dbPath == "" {
|
||||
return nil, fmt.Errorf("%s: file path or connection string is required for SQLite format", label)
|
||||
}
|
||||
reader = sqlite.NewReader(&readers.ReaderOptions{FilePath: dbPath})
|
||||
reader = sqlite.NewReader(newReaderOptions(dbPath, ""))
|
||||
|
||||
default:
|
||||
return nil, fmt.Errorf("%s: unsupported database format: %s", label, dbType)
|
||||
|
||||
@@ -0,0 +1,158 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
|
||||
func newDryRunCmd() (*cobra.Command, *bytes.Buffer) {
|
||||
var buf bytes.Buffer
|
||||
cmd := &cobra.Command{}
|
||||
cmd.SetOut(&buf)
|
||||
return cmd, &buf
|
||||
}
|
||||
|
||||
func TestRunConvert_DryRunWritesNothing(t *testing.T) {
|
||||
defer func(a, b, c, d string, e bool) {
|
||||
convertSourceType, convertSourcePath, convertTargetType, convertTargetPath, convertDryRun = a, b, c, d, e
|
||||
}(convertSourceType, convertSourcePath, convertTargetType, convertTargetPath, convertDryRun)
|
||||
|
||||
dir := t.TempDir()
|
||||
in := filepath.Join(dir, "in.json")
|
||||
out := filepath.Join(dir, "out.json")
|
||||
writeTestJSON(t, in, []string{"users", "posts"})
|
||||
|
||||
convertSourceType, convertSourcePath = "json", in
|
||||
convertTargetType, convertTargetPath = "json", out
|
||||
convertDryRun = true
|
||||
|
||||
cmd, buf := newDryRunCmd()
|
||||
if err := runConvert(cmd, nil); err != nil {
|
||||
t.Fatalf("dry run: %v", err)
|
||||
}
|
||||
if _, err := os.Stat(out); !os.IsNotExist(err) {
|
||||
t.Fatal("dry run must not create the output file")
|
||||
}
|
||||
for _, want := range []string{"dry run", "users", "posts", out} {
|
||||
if !strings.Contains(buf.String(), want) {
|
||||
t.Errorf("plan missing %q:\n%s", want, buf.String())
|
||||
}
|
||||
}
|
||||
|
||||
// Normal behavior is unchanged.
|
||||
convertDryRun = false
|
||||
if err := runConvert(cmd, nil); err != nil {
|
||||
t.Fatalf("real run: %v", err)
|
||||
}
|
||||
if _, err := os.Stat(out); err != nil {
|
||||
t.Fatalf("real run should write output: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunConvert_DryRunValidatesTarget(t *testing.T) {
|
||||
defer func(a, b, c, d string, e bool) {
|
||||
convertSourceType, convertSourcePath, convertTargetType, convertTargetPath, convertDryRun = a, b, c, d, e
|
||||
}(convertSourceType, convertSourcePath, convertTargetType, convertTargetPath, convertDryRun)
|
||||
|
||||
dir := t.TempDir()
|
||||
in := filepath.Join(dir, "in.json")
|
||||
out := filepath.Join(dir, "models")
|
||||
writeTestJSON(t, in, []string{"users"})
|
||||
|
||||
convertSourceType, convertSourcePath = "json", in
|
||||
convertDryRun = true
|
||||
|
||||
// gorm without --package must fail validation, as a real run would.
|
||||
convertTargetType, convertTargetPath = "gorm", out
|
||||
cmd, _ := newDryRunCmd()
|
||||
err := runConvert(cmd, nil)
|
||||
if err == nil || !strings.Contains(err.Error(), "package name is required") {
|
||||
t.Fatalf("expected package validation error, got %v", err)
|
||||
}
|
||||
|
||||
convertTargetType = "nope"
|
||||
err = runConvert(cmd, nil)
|
||||
if err == nil || !strings.Contains(err.Error(), "unsupported target format") {
|
||||
t.Fatalf("expected unsupported format error, got %v", err)
|
||||
}
|
||||
if _, statErr := os.Stat(out); !os.IsNotExist(statErr) {
|
||||
t.Fatal("dry run must not create the output path")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunSplit_DryRunWritesNothing(t *testing.T) {
|
||||
defer func(a, b, c, d, e string, f bool) {
|
||||
splitSourceType, splitSourcePath, splitTargetType, splitTargetPath, splitTables, splitDryRun = a, b, c, d, e, f
|
||||
}(splitSourceType, splitSourcePath, splitTargetType, splitTargetPath, splitTables, splitDryRun)
|
||||
|
||||
dir := t.TempDir()
|
||||
in := filepath.Join(dir, "in.json")
|
||||
out := filepath.Join(dir, "subset.json")
|
||||
writeTestJSON(t, in, []string{"users", "posts", "comments"})
|
||||
|
||||
splitSourceType, splitSourcePath = "json", in
|
||||
splitTargetType, splitTargetPath = "json", out
|
||||
splitTables = "users,posts"
|
||||
splitDryRun = true
|
||||
|
||||
cmd, buf := newDryRunCmd()
|
||||
if err := runSplit(cmd, nil); err != nil {
|
||||
t.Fatalf("dry run: %v", err)
|
||||
}
|
||||
if _, err := os.Stat(out); !os.IsNotExist(err) {
|
||||
t.Fatal("dry run must not create the output file")
|
||||
}
|
||||
got := buf.String()
|
||||
if !strings.Contains(got, "2 table(s)") || strings.Contains(got, "comments") {
|
||||
t.Errorf("plan should show only the 2 selected tables:\n%s", got)
|
||||
}
|
||||
|
||||
// A selection that matches nothing fails validation in dry-run too.
|
||||
splitTables = "does_not_exist"
|
||||
if err := runSplit(cmd, nil); err == nil {
|
||||
t.Fatal("expected error for empty selection")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunMerge_DryRunWritesNothing(t *testing.T) {
|
||||
saved := saveMergeState()
|
||||
defer restoreMergeState(saved)
|
||||
defer func(v bool) { mergeDryRun = v }(mergeDryRun)
|
||||
|
||||
dir := t.TempDir()
|
||||
target := filepath.Join(dir, "target.json")
|
||||
source := filepath.Join(dir, "source.json")
|
||||
out := filepath.Join(dir, "merged.json")
|
||||
writeTestJSON(t, target, []string{"users"})
|
||||
writeTestJSON(t, source, []string{"posts"})
|
||||
|
||||
mergeTargetType, mergeTargetPath, mergeTargetConn = "json", target, ""
|
||||
mergeSourceType, mergeSourcePath, mergeSourceConn = "json", source, ""
|
||||
mergeFromList = nil
|
||||
mergeOutputType, mergeOutputPath, mergeOutputConn = "json", out, ""
|
||||
mergeSkipTables, mergeReportPath = "", ""
|
||||
mergeDryRun = true
|
||||
|
||||
cmd, buf := newDryRunCmd()
|
||||
if err := runMerge(cmd, nil); err != nil {
|
||||
t.Fatalf("dry run: %v", err)
|
||||
}
|
||||
if _, err := os.Stat(out); !os.IsNotExist(err) {
|
||||
t.Fatal("dry run must not create the output file")
|
||||
}
|
||||
for _, want := range []string{"dry run", "users", "posts", out} {
|
||||
if !strings.Contains(buf.String(), want) {
|
||||
t.Errorf("plan missing %q:\n%s", want, buf.String())
|
||||
}
|
||||
}
|
||||
|
||||
mergeOutputType = "nope"
|
||||
if err := runMerge(cmd, nil); err == nil || !strings.Contains(err.Error(), "unsupported format") {
|
||||
t.Fatalf("expected unsupported output format error, got %v", err)
|
||||
}
|
||||
}
|
||||
@@ -193,7 +193,7 @@ func runInspect(cmd *cobra.Command, args []string) error {
|
||||
|
||||
// Write output
|
||||
if inspectOutputPath != "" {
|
||||
err = os.WriteFile(inspectOutputPath, []byte(formattedReport), 0644)
|
||||
err = os.WriteFile(inspectOutputPath, []byte(formattedReport), 0o644)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to write output file: %w", err)
|
||||
}
|
||||
|
||||
+427
-21
@@ -1,6 +1,7 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
@@ -11,12 +12,17 @@ import (
|
||||
|
||||
"github.com/spf13/cobra"
|
||||
|
||||
"git.warky.dev/wdevs/relspecgo/pkg/diff"
|
||||
"git.warky.dev/wdevs/relspecgo/pkg/inspector"
|
||||
"git.warky.dev/wdevs/relspecgo/pkg/jobs"
|
||||
"git.warky.dev/wdevs/relspecgo/pkg/merge"
|
||||
"git.warky.dev/wdevs/relspecgo/pkg/models"
|
||||
"git.warky.dev/wdevs/relspecgo/pkg/readers"
|
||||
"git.warky.dev/wdevs/relspecgo/pkg/readers/sqldir"
|
||||
"git.warky.dev/wdevs/relspecgo/pkg/writers"
|
||||
wpgsql "git.warky.dev/wdevs/relspecgo/pkg/writers/pgsql"
|
||||
"git.warky.dev/wdevs/relspecgo/pkg/writers/sqlexec"
|
||||
wtemplate "git.warky.dev/wdevs/relspecgo/pkg/writers/template"
|
||||
)
|
||||
|
||||
var (
|
||||
@@ -54,8 +60,9 @@ inputs, output and options:
|
||||
logfile: .relspec/log/build-schema.log
|
||||
|
||||
Rules and guarantees:
|
||||
- command is a closed allow-list (convert, merge, scripts-list). Arbitrary
|
||||
shell strings are never executed.
|
||||
- command is a closed allow-list (convert, merge, split, scripts-list,
|
||||
scripts-exec, templ, inspect, diff). Arbitrary shell strings are never
|
||||
executed.
|
||||
- Every path is relative to the directory holding the job file and may not
|
||||
escape it. Absolute and home-relative paths are rejected.
|
||||
- Remote database credentials are referenced by environment-variable name
|
||||
@@ -120,6 +127,9 @@ func loadJobSet() (*jobs.Set, error) {
|
||||
if err := set.Validate(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, w := range set.Warnings {
|
||||
fmt.Fprintf(os.Stderr, "warning: %s\n", w)
|
||||
}
|
||||
return set, nil
|
||||
}
|
||||
|
||||
@@ -178,12 +188,14 @@ func executeJobPlan(set *jobs.Set, name string, dryRun, noDeps bool, out io.Writ
|
||||
// Pre-flight: resolve and check paths, output policy and env vars for the
|
||||
// whole plan before anything runs. A failure here means no job executes.
|
||||
resolved := make([]*resolvedJob, len(plan))
|
||||
byName := make(map[string]*resolvedJob, len(plan))
|
||||
for i, j := range plan {
|
||||
rj, perr := preflightJob(j)
|
||||
rj, perr := preflightJob(j, byName)
|
||||
if perr != nil {
|
||||
return fmt.Errorf("job %q: %w", j.Name, perr)
|
||||
}
|
||||
resolved[i] = rj
|
||||
byName[j.Name] = rj
|
||||
}
|
||||
|
||||
if dryRun {
|
||||
@@ -215,6 +227,12 @@ type resolvedJob struct {
|
||||
outputConn string // resolved connection string (secret)
|
||||
outputConnEnv string
|
||||
logPath string
|
||||
logPolicy jobs.LogPolicy
|
||||
templatePath string
|
||||
reportPath string // "" for a diff summary written to the log
|
||||
reportFormat string
|
||||
rulesPath string // "" means inspector defaults
|
||||
selection *splitSelection
|
||||
secrets []string // resolved secret values to redact from logs
|
||||
}
|
||||
|
||||
@@ -223,22 +241,63 @@ type resolvedInput struct {
|
||||
path string // "" when the input is a database
|
||||
conn string // resolved connection string (secret)
|
||||
connEnv string
|
||||
fromJob string // producer job name when this input came from from_job
|
||||
}
|
||||
|
||||
func preflightJob(j *jobs.Job) (*resolvedJob, error) {
|
||||
func preflightJob(j *jobs.Job, resolvedByName map[string]*resolvedJob) (*resolvedJob, error) {
|
||||
root := j.Dir()
|
||||
rj := &resolvedJob{job: j, root: root}
|
||||
rj := &resolvedJob{job: j, root: root, logPolicy: j.ResolvedLogPolicy()}
|
||||
expandPath := func(label, value string) (string, error) {
|
||||
expanded, err := jobs.ExpandEnv(value)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("%s: %w", label, err)
|
||||
}
|
||||
return expanded, nil
|
||||
}
|
||||
|
||||
if j.Logfile != "" {
|
||||
p, err := jobs.SafeJoin(root, j.Logfile)
|
||||
logfile, err := expandPath("logfile", j.Logfile)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
p, err := jobs.SafeJoin(root, logfile)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("logfile: %w", err)
|
||||
}
|
||||
rj.logPath = p
|
||||
}
|
||||
if j.Template != "" {
|
||||
templatePath, err := expandPath("template", j.Template)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
p, err := jobs.SafeJoin(root, templatePath)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("template: %w", err)
|
||||
}
|
||||
info, err := os.Stat(p)
|
||||
if err != nil || info.IsDir() {
|
||||
return nil, fmt.Errorf("template %q: not found or is a directory", templatePath)
|
||||
}
|
||||
rj.templatePath = p
|
||||
}
|
||||
|
||||
for i, in := range j.Inputs {
|
||||
ri := resolvedInput{format: strings.ToLower(in.Format)}
|
||||
if in.FromJob != "" {
|
||||
producer, ok := resolvedByName[in.FromJob]
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("input[%d]: from_job %q is not in this plan (do not use --no-deps with from_job inputs)", i, in.FromJob)
|
||||
}
|
||||
if producer.outputPath == "" {
|
||||
return nil, fmt.Errorf("input[%d]: from_job %q does not write a file output", i, in.FromJob)
|
||||
}
|
||||
ri.path = producer.outputPath
|
||||
ri.format = strings.ToLower(producer.job.Output.Format)
|
||||
ri.fromJob = in.FromJob
|
||||
rj.inputs = append(rj.inputs, ri)
|
||||
continue
|
||||
}
|
||||
if in.ConnEnv != "" {
|
||||
v, ok := os.LookupEnv(in.ConnEnv)
|
||||
if !ok || v == "" {
|
||||
@@ -248,16 +307,20 @@ func preflightJob(j *jobs.Job) (*resolvedJob, error) {
|
||||
ri.connEnv = in.ConnEnv
|
||||
rj.secrets = append(rj.secrets, v)
|
||||
} else {
|
||||
p, err := jobs.SafeJoin(root, in.Path)
|
||||
inputPath, err := expandPath(fmt.Sprintf("input[%d]", i), in.Path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
p, err := jobs.SafeJoin(root, inputPath)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("input[%d]: %w", i, err)
|
||||
}
|
||||
info, err := os.Stat(p)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("input[%d]: %s: file not found", i, in.Path)
|
||||
return nil, fmt.Errorf("input[%d]: %s: file not found", i, inputPath)
|
||||
}
|
||||
if info.IsDir() {
|
||||
return nil, fmt.Errorf("input[%d]: %s: is a directory, not a file", i, in.Path)
|
||||
return nil, fmt.Errorf("input[%d]: %s: is a directory, not a file", i, inputPath)
|
||||
}
|
||||
ri.path = p
|
||||
}
|
||||
@@ -265,13 +328,17 @@ func preflightJob(j *jobs.Job) (*resolvedJob, error) {
|
||||
}
|
||||
|
||||
for _, d := range j.ScriptDirs {
|
||||
p, err := jobs.SafeJoin(root, d)
|
||||
scriptDir, err := expandPath("script_dir", d)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
p, err := jobs.SafeJoin(root, scriptDir)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("script_dir %q: %w", d, err)
|
||||
}
|
||||
info, err := os.Stat(p)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("script_dir %q: not found", d)
|
||||
return nil, fmt.Errorf("script_dir %q: not found", scriptDir)
|
||||
}
|
||||
if !info.IsDir() {
|
||||
return nil, fmt.Errorf("script_dir %q: not a directory", d)
|
||||
@@ -289,16 +356,65 @@ func preflightJob(j *jobs.Job) (*resolvedJob, error) {
|
||||
rj.outputConnEnv = j.Output.ConnEnv
|
||||
rj.secrets = append(rj.secrets, v)
|
||||
} else {
|
||||
p, err := jobs.SafeJoin(root, j.Output.Path)
|
||||
outputPath, err := expandPath("output", j.Output.Path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
p, err := jobs.SafeJoin(root, outputPath)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("output: %w", err)
|
||||
}
|
||||
if _, err := os.Stat(p); err == nil && !j.Output.Overwrite {
|
||||
return nil, fmt.Errorf("output %s already exists (set output.overwrite: true to replace it)", j.Output.Path)
|
||||
return nil, fmt.Errorf("output %s already exists (set output.overwrite: true to replace it)", outputPath)
|
||||
}
|
||||
rj.outputPath = p
|
||||
}
|
||||
}
|
||||
|
||||
if j.Rules != "" {
|
||||
rulesPath, err := expandPath("rules", j.Rules)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
p, err := jobs.SafeJoin(root, rulesPath)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("rules: %w", err)
|
||||
}
|
||||
info, err := os.Stat(p)
|
||||
if err != nil || info.IsDir() {
|
||||
return nil, fmt.Errorf("rules %q: not found or is a directory", rulesPath)
|
||||
}
|
||||
rj.rulesPath = p
|
||||
}
|
||||
|
||||
if j.Report != nil {
|
||||
rj.reportFormat = strings.ToLower(j.Report.Format)
|
||||
if j.Report.Path != "" {
|
||||
reportPath, err := expandPath("report", j.Report.Path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
p, err := jobs.SafeJoin(root, reportPath)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("report: %w", err)
|
||||
}
|
||||
if _, err := os.Stat(p); err == nil && !j.Report.Overwrite {
|
||||
return nil, fmt.Errorf("report %s already exists (set report.overwrite: true to replace it)", reportPath)
|
||||
}
|
||||
rj.reportPath = p
|
||||
}
|
||||
}
|
||||
|
||||
if j.Select != nil {
|
||||
rj.selection = &splitSelection{
|
||||
Schemas: j.Select.Schemas,
|
||||
Tables: j.Select.Tables,
|
||||
ExcludeSchemas: j.Select.ExcludeSchemas,
|
||||
ExcludeTables: j.Select.ExcludeTables,
|
||||
DatabaseName: j.Select.DatabaseName,
|
||||
}
|
||||
}
|
||||
|
||||
return rj, nil
|
||||
}
|
||||
|
||||
@@ -311,9 +427,12 @@ func printResolvedJob(out io.Writer, n, total int, rj *resolvedJob) {
|
||||
}
|
||||
fmt.Fprintf(out, " job file: %s\n", j.SourceFile)
|
||||
for _, ri := range rj.inputs {
|
||||
if ri.path != "" {
|
||||
switch {
|
||||
case ri.fromJob != "":
|
||||
fmt.Fprintf(out, " input: %s (%s) from job %q\n", ri.path, ri.format, ri.fromJob)
|
||||
case ri.path != "":
|
||||
fmt.Fprintf(out, " input: %s (%s)\n", ri.path, ri.format)
|
||||
} else {
|
||||
default:
|
||||
fmt.Fprintf(out, " input: env:%s (%s)\n", ri.connEnv, ri.format)
|
||||
}
|
||||
}
|
||||
@@ -325,15 +444,31 @@ func printResolvedJob(out io.Writer, n, total int, rj *resolvedJob) {
|
||||
} else if rj.outputConnEnv != "" {
|
||||
fmt.Fprintf(out, " output: env:%s (%s)\n", rj.outputConnEnv, j.Output.Format)
|
||||
}
|
||||
if j.Report != nil {
|
||||
format := valueOr(rj.reportFormat, "default")
|
||||
if rj.reportPath != "" {
|
||||
fmt.Fprintf(out, " report: %s (%s)\n", rj.reportPath, format)
|
||||
} else {
|
||||
fmt.Fprintf(out, " report: (log) (%s)\n", format)
|
||||
}
|
||||
}
|
||||
if rj.rulesPath != "" {
|
||||
fmt.Fprintf(out, " rules: %s\n", rj.rulesPath)
|
||||
} else if j.Command == jobs.CommandInspect {
|
||||
fmt.Fprintf(out, " rules: (built-in defaults)\n")
|
||||
}
|
||||
if rj.selection != nil {
|
||||
fmt.Fprintf(out, " select: %s\n", rj.selection.summary())
|
||||
}
|
||||
if rj.logPath != "" {
|
||||
fmt.Fprintf(out, " logfile: %s\n", rj.logPath)
|
||||
fmt.Fprintf(out, " logfile: %s (rotate >= %d bytes, keep %d)\n", rj.logPath, rj.logPolicy.MaxSizeBytes, rj.logPolicy.Keep)
|
||||
}
|
||||
fmt.Fprintln(out)
|
||||
}
|
||||
|
||||
// executeResolvedJob runs a single already-validated job.
|
||||
func executeResolvedJob(rj *resolvedJob) (err error) {
|
||||
lg, closeLog, lerr := newJobLogger(rj.logPath, rj.secrets)
|
||||
lg, closeLog, lerr := newJobLogger(rj.logPath, rj.logPolicy, rj.secrets)
|
||||
if lerr != nil {
|
||||
return lerr
|
||||
}
|
||||
@@ -348,6 +483,16 @@ func executeResolvedJob(rj *resolvedJob) (err error) {
|
||||
err = runMergeJob(rj, lg)
|
||||
case jobs.CommandScriptsList:
|
||||
err = runScriptsListJob(rj, lg)
|
||||
case jobs.CommandTempl:
|
||||
err = runTemplJob(rj, lg)
|
||||
case jobs.CommandSplit:
|
||||
err = runSplitJob(rj, lg)
|
||||
case jobs.CommandInspect:
|
||||
err = runInspectJob(rj, lg)
|
||||
case jobs.CommandDiff:
|
||||
err = runDiffJob(rj, lg)
|
||||
case jobs.CommandScriptsExec:
|
||||
err = runScriptsExecJob(rj, lg)
|
||||
default:
|
||||
err = fmt.Errorf("unsupported command %q", rj.job.Command)
|
||||
}
|
||||
@@ -359,6 +504,50 @@ func executeResolvedJob(rj *resolvedJob) (err error) {
|
||||
return err
|
||||
}
|
||||
|
||||
func runTemplJob(rj *resolvedJob, lg *jobLogger) error {
|
||||
db, err := readJobInputs(rj, lg)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if schema := rj.job.Options.Schema; schema != "" {
|
||||
found := false
|
||||
for _, s := range db.Schemas {
|
||||
if s.Name == schema {
|
||||
db.Schemas = []*models.Schema{s}
|
||||
found = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
return fmt.Errorf("schema not found: %s", schema)
|
||||
}
|
||||
}
|
||||
mode := rj.job.Mode
|
||||
if mode == "" {
|
||||
mode = "database"
|
||||
}
|
||||
pattern := rj.job.FilenamePattern
|
||||
if pattern == "" {
|
||||
pattern = "{{.Name}}.txt"
|
||||
}
|
||||
writer, err := wtemplate.NewWriter(&writers.WriterOptions{
|
||||
OutputPath: rj.outputPath,
|
||||
Metadata: map[string]interface{}{
|
||||
"template_path": rj.templatePath,
|
||||
"mode": mode,
|
||||
"filename_pattern": pattern,
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("create template writer: %w", err)
|
||||
}
|
||||
lg.logf("applying template: %s (mode %s)", rj.templatePath, mode)
|
||||
if err := writer.WriteDatabase(db); err != nil {
|
||||
return fmt.Errorf("execute template: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func runConvertJob(rj *resolvedJob, lg *jobLogger) error {
|
||||
db, err := readJobInputs(rj, lg)
|
||||
if err != nil {
|
||||
@@ -389,6 +578,7 @@ func runMergeJob(rj *resolvedJob, lg *jobLogger) error {
|
||||
lg.logf("merging: %s", inputLabel(ri))
|
||||
merge.MergeDatabases(base, db, opts)
|
||||
}
|
||||
finalizeCommentedRefs(base, func(w string) { lg.logf("warning: %s", w) })
|
||||
base.UpdateDate()
|
||||
return writeJobOutput(rj, base, lg)
|
||||
}
|
||||
@@ -445,6 +635,168 @@ func runScriptsListJob(rj *resolvedJob, lg *jobLogger) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func runSplitJob(rj *resolvedJob, lg *jobLogger) error {
|
||||
db, err := readJobInputs(rj, lg)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
sel := splitSelection{}
|
||||
if rj.selection != nil {
|
||||
sel = *rj.selection
|
||||
}
|
||||
filtered, err := filterDatabaseSelection(db, sel)
|
||||
if err != nil {
|
||||
return fmt.Errorf("split selection: %w", err)
|
||||
}
|
||||
if sel.DatabaseName != "" {
|
||||
filtered.Name = sel.DatabaseName
|
||||
}
|
||||
tables := 0
|
||||
for _, s := range filtered.Schemas {
|
||||
tables += len(s.Tables)
|
||||
}
|
||||
lg.logf("split: selected %d schema(s), %d table(s)", len(filtered.Schemas), tables)
|
||||
return writeJobOutput(rj, filtered, lg)
|
||||
}
|
||||
|
||||
func runInspectJob(rj *resolvedJob, lg *jobLogger) error {
|
||||
db, err := readJobInputs(rj, lg)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
config, err := inspector.LoadConfig(rj.rulesPath) // "" -> built-in defaults
|
||||
if err != nil {
|
||||
return fmt.Errorf("load rules: %w", err)
|
||||
}
|
||||
report, err := inspector.NewInspector(db, config).Inspect()
|
||||
if err != nil {
|
||||
return fmt.Errorf("inspection failed: %w", err)
|
||||
}
|
||||
|
||||
var formatted string
|
||||
switch valueOr(rj.reportFormat, "markdown") {
|
||||
case "json":
|
||||
formatted, err = inspector.NewJSONFormatter().Format(report)
|
||||
default:
|
||||
formatted, err = inspector.NewMarkdownFormatter(io.Discard).Format(report)
|
||||
}
|
||||
if err != nil {
|
||||
return fmt.Errorf("format report: %w", err)
|
||||
}
|
||||
if werr := atomicWrite(rj.reportPath, func(tmp string) error {
|
||||
return os.WriteFile(tmp, []byte(formatted), 0o644)
|
||||
}); werr != nil {
|
||||
return werr
|
||||
}
|
||||
lg.logf("inspect: %d error(s), %d warning(s) -> %s",
|
||||
report.Summary.ErrorCount, report.Summary.WarningCount, rj.reportPath)
|
||||
if report.HasErrors() {
|
||||
return fmt.Errorf("inspection found %d error(s)", report.Summary.ErrorCount)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func runDiffJob(rj *resolvedJob, lg *jobLogger) error {
|
||||
if len(rj.inputs) != 2 {
|
||||
return fmt.Errorf("diff requires exactly 2 inputs, got %d", len(rj.inputs))
|
||||
}
|
||||
source, err := readOneJobInput(rj.inputs[0])
|
||||
if err != nil {
|
||||
return fmt.Errorf("input[0]: %w", err)
|
||||
}
|
||||
lg.logf("diff source: %s", inputLabel(rj.inputs[0]))
|
||||
target, err := readOneJobInput(rj.inputs[1])
|
||||
if err != nil {
|
||||
return fmt.Errorf("input[1]: %w", err)
|
||||
}
|
||||
lg.logf("diff target: %s", inputLabel(rj.inputs[1]))
|
||||
|
||||
result := diff.CompareDatabases(source, target)
|
||||
s := diff.ComputeSummary(result)
|
||||
lg.logf("diff: schemas %d/%d/%d, tables %d/%d/%d, columns %d/%d/%d (missing/extra/modified)",
|
||||
s.Schemas.Missing, s.Schemas.Extra, s.Schemas.Modified,
|
||||
s.Tables.Missing, s.Tables.Extra, s.Tables.Modified,
|
||||
s.Columns.Missing, s.Columns.Extra, s.Columns.Modified)
|
||||
|
||||
format := diff.FormatSummary
|
||||
switch rj.reportFormat {
|
||||
case "json":
|
||||
format = diff.FormatJSON
|
||||
case "html":
|
||||
format = diff.FormatHTML
|
||||
}
|
||||
|
||||
if rj.reportPath == "" {
|
||||
var buf bytes.Buffer
|
||||
if err := diff.FormatDiff(result, format, &buf); err != nil {
|
||||
return fmt.Errorf("format diff: %w", err)
|
||||
}
|
||||
for _, line := range strings.Split(strings.TrimRight(buf.String(), "\n"), "\n") {
|
||||
lg.logf("%s", line)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
if werr := atomicWrite(rj.reportPath, func(tmp string) error {
|
||||
f, err := os.Create(tmp)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer f.Close()
|
||||
return diff.FormatDiff(result, format, f)
|
||||
}); werr != nil {
|
||||
return werr
|
||||
}
|
||||
lg.logf("diff report written: %s", rj.reportPath)
|
||||
return nil
|
||||
}
|
||||
|
||||
func runScriptsExecJob(rj *resolvedJob, lg *jobLogger) error {
|
||||
schemaName := valueOr(rj.job.Options.Schema, "public")
|
||||
combined := &models.Schema{Name: schemaName}
|
||||
for _, dir := range rj.scriptDirs {
|
||||
reader := sqldir.NewReader(&readers.ReaderOptions{
|
||||
FilePath: dir,
|
||||
Metadata: map[string]any{
|
||||
"schema_name": schemaName,
|
||||
"database_name": "database",
|
||||
},
|
||||
})
|
||||
db, err := reader.ReadDatabase()
|
||||
if err != nil {
|
||||
return fmt.Errorf("%s: %w", dir, err)
|
||||
}
|
||||
if len(db.Schemas) == 0 {
|
||||
continue
|
||||
}
|
||||
combined.Scripts = append(combined.Scripts, db.Schemas[0].Scripts...)
|
||||
}
|
||||
if len(combined.Scripts) == 0 {
|
||||
lg.logf("no scripts found; nothing to execute")
|
||||
return nil
|
||||
}
|
||||
lg.logf("executing %d script(s) against database env:%s", len(combined.Scripts), rj.outputConnEnv)
|
||||
|
||||
writer := sqlexec.NewWriter(&writers.WriterOptions{
|
||||
Metadata: map[string]any{
|
||||
"connection_string": rj.outputConn,
|
||||
"ignore_errors": rj.job.Options.ContinueOnError,
|
||||
},
|
||||
})
|
||||
if err := writer.WriteSchema(combined); err != nil {
|
||||
return fmt.Errorf("script execution failed: %w", err)
|
||||
}
|
||||
|
||||
opts := writer.Options()
|
||||
total, _ := opts.Metadata["execution_total"].(int)
|
||||
success, _ := opts.Metadata["execution_success"].(int)
|
||||
failed, _ := opts.Metadata["execution_failed"].(int)
|
||||
lg.logf("executed %d script(s): %d succeeded, %d failed", total, success, failed)
|
||||
if failed > 0 && !rj.job.Options.ContinueOnError {
|
||||
return fmt.Errorf("%d script(s) failed", failed)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// readJobInputs reads every input and additively merges them into one model.
|
||||
func readJobInputs(rj *resolvedJob, lg *jobLogger) (*models.Database, error) {
|
||||
var base *models.Database
|
||||
@@ -463,6 +815,7 @@ func readJobInputs(rj *resolvedJob, lg *jobLogger) (*models.Database, error) {
|
||||
if base == nil {
|
||||
return nil, fmt.Errorf("no inputs produced a database")
|
||||
}
|
||||
finalizeCommentedRefs(base, func(w string) { lg.logf("warning: %s", w) })
|
||||
return base, nil
|
||||
}
|
||||
|
||||
@@ -490,8 +843,8 @@ func writeJobOutput(rj *resolvedJob, db *models.Database, lg *jobLogger) error {
|
||||
return fmt.Errorf("database output is only supported for pgsql (got %q)", rj.job.Output.Format)
|
||||
}
|
||||
lg.logf("writing output to database env:%s", rj.outputConnEnv)
|
||||
writerOpts := newWriterOptions("", o.Package, o.FlattenSchema, "", "", o.ContinueOnError)
|
||||
writerOpts.Metadata = map[string]interface{}{"connection_string": rj.outputConn}
|
||||
writerOpts := newWriterOptions("", o.Package, o.FlattenSchema, o.Types, o.ArrayNullable, o.ContinueOnError)
|
||||
writerOpts.Metadata = map[string]interface{}{"connection_string": rj.outputConn, "full_ddl": o.FullDDL}
|
||||
return wpgsql.NewWriter(writerOpts).WriteDatabase(db)
|
||||
}
|
||||
|
||||
@@ -499,7 +852,37 @@ func writeJobOutput(rj *resolvedJob, db *models.Database, lg *jobLogger) error {
|
||||
return fmt.Errorf("failed to create output directory: %w", err)
|
||||
}
|
||||
lg.logf("writing output: %s (%s)", rj.outputPath, format)
|
||||
return writeDatabase(db, format, rj.outputPath, o.Package, o.Schema, o.FlattenSchema, "", "", o.ContinueOnError, "")
|
||||
|
||||
write := func(target string) error {
|
||||
return writeDatabase(db, format, target, o.Package, o.Schema, o.FlattenSchema, o.Types, o.ArrayNullable, o.ContinueOnError, "")
|
||||
}
|
||||
// Single-file formats are written to a temp file and renamed into place so
|
||||
// a failure never leaves a partial or truncated output. Directory-emitting
|
||||
// formats (gorm/bun/drizzle/typeorm/prisma) write in place.
|
||||
if jobs.SingleFileOutputFormat(format) {
|
||||
return atomicWrite(rj.outputPath, write)
|
||||
}
|
||||
return write(rj.outputPath)
|
||||
}
|
||||
|
||||
// atomicWrite calls produce with a temp path in the same directory as
|
||||
// finalPath, then renames it over finalPath. The temp file is removed on any
|
||||
// error so the destination is only ever replaced by a complete file.
|
||||
func atomicWrite(finalPath string, produce func(tmpPath string) error) error {
|
||||
dir := filepath.Dir(finalPath)
|
||||
if err := os.MkdirAll(dir, 0o755); err != nil {
|
||||
return fmt.Errorf("failed to create output directory: %w", err)
|
||||
}
|
||||
tmp := filepath.Join(dir, fmt.Sprintf(".%s.relspec-tmp-%d", filepath.Base(finalPath), os.Getpid()))
|
||||
if err := produce(tmp); err != nil {
|
||||
_ = os.Remove(tmp)
|
||||
return err
|
||||
}
|
||||
if err := os.Rename(tmp, finalPath); err != nil {
|
||||
_ = os.Remove(tmp)
|
||||
return fmt.Errorf("failed to finalize %s: %w", finalPath, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// --- logging + redaction ---------------------------------------------------
|
||||
@@ -512,7 +895,7 @@ type jobLogger struct {
|
||||
// newJobLogger returns a logger that mirrors to stderr and, when path is set,
|
||||
// to a job logfile. Connection strings and known secret values are redacted
|
||||
// from everything it writes.
|
||||
func newJobLogger(path string, secrets []string) (*jobLogger, func(err error), error) {
|
||||
func newJobLogger(path string, policy jobs.LogPolicy, secrets []string) (*jobLogger, func(err error), error) {
|
||||
lg := &jobLogger{secrets: secrets}
|
||||
if path == "" {
|
||||
return lg, func(error) {}, nil
|
||||
@@ -520,6 +903,7 @@ func newJobLogger(path string, secrets []string) (*jobLogger, func(err error), e
|
||||
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
|
||||
return nil, nil, fmt.Errorf("failed to create log directory: %w", err)
|
||||
}
|
||||
rotateLogIfNeeded(path, policy)
|
||||
f, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0o644)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("failed to open logfile %q: %w", path, err)
|
||||
@@ -533,6 +917,28 @@ func newJobLogger(path string, secrets []string) (*jobLogger, func(err error), e
|
||||
}, nil
|
||||
}
|
||||
|
||||
// rotateLogIfNeeded renames path -> path.1 -> path.2 ... up to policy.Keep
|
||||
// when path has grown to policy.MaxSizeBytes or more. The oldest file beyond
|
||||
// Keep is deleted. A zero/negative MaxSizeBytes disables rotation.
|
||||
func rotateLogIfNeeded(path string, policy jobs.LogPolicy) {
|
||||
if policy.MaxSizeBytes <= 0 {
|
||||
return
|
||||
}
|
||||
info, err := os.Stat(path)
|
||||
if err != nil || info.Size() < policy.MaxSizeBytes {
|
||||
return
|
||||
}
|
||||
if policy.Keep < 1 {
|
||||
_ = os.Remove(path)
|
||||
return
|
||||
}
|
||||
_ = os.Remove(fmt.Sprintf("%s.%d", path, policy.Keep))
|
||||
for i := policy.Keep - 1; i >= 1; i-- {
|
||||
_ = os.Rename(fmt.Sprintf("%s.%d", path, i), fmt.Sprintf("%s.%d", path, i+1))
|
||||
}
|
||||
_ = os.Rename(path, path+".1")
|
||||
}
|
||||
|
||||
func (l *jobLogger) logf(format string, args ...interface{}) {
|
||||
line := l.redact(fmt.Sprintf(format, args...))
|
||||
fmt.Fprintf(os.Stderr, " %s\n", line)
|
||||
|
||||
@@ -375,3 +375,369 @@ jobs:
|
||||
t.Fatal("job list output not deterministic")
|
||||
}
|
||||
}
|
||||
|
||||
func TestJobRun_SplitJob(t *testing.T) {
|
||||
dir := jobFixture(t, `version: 1
|
||||
jobs:
|
||||
extract:
|
||||
command: split
|
||||
inputs:
|
||||
- path: schema/core.dbml
|
||||
format: dbml
|
||||
- path: schema/tenant.dbml
|
||||
format: dbml
|
||||
select:
|
||||
tables: [users]
|
||||
output:
|
||||
format: json
|
||||
path: build/subset.json
|
||||
overwrite: true
|
||||
`)
|
||||
set := mustLoadSet(t, filepath.Join(dir, "relspec.yml"))
|
||||
if err := executeJobPlan(set, "extract", false, false, &bytes.Buffer{}); err != nil {
|
||||
t.Fatalf("execute split job: %v", err)
|
||||
}
|
||||
out, err := os.ReadFile(filepath.Join(dir, "build", "subset.json"))
|
||||
if err != nil {
|
||||
t.Fatalf("read split output: %v", err)
|
||||
}
|
||||
s := string(out)
|
||||
if !strings.Contains(s, "users") {
|
||||
t.Fatalf("split output missing selected table:\n%s", s)
|
||||
}
|
||||
if strings.Contains(s, "posts") {
|
||||
t.Fatalf("split output should have excluded posts:\n%s", s)
|
||||
}
|
||||
}
|
||||
|
||||
func TestJobRun_InspectJob(t *testing.T) {
|
||||
dir := jobFixture(t, `version: 1
|
||||
jobs:
|
||||
lint:
|
||||
command: inspect
|
||||
inputs:
|
||||
- path: schema/core.dbml
|
||||
format: dbml
|
||||
report:
|
||||
format: json
|
||||
path: build/report.json
|
||||
overwrite: true
|
||||
logfile: .relspec/lint.log
|
||||
`)
|
||||
set := mustLoadSet(t, filepath.Join(dir, "relspec.yml"))
|
||||
// Default rules only warn, so the job succeeds.
|
||||
if err := executeJobPlan(set, "lint", false, false, &bytes.Buffer{}); err != nil {
|
||||
t.Fatalf("execute inspect job: %v", err)
|
||||
}
|
||||
if _, err := os.ReadFile(filepath.Join(dir, "build", "report.json")); err != nil {
|
||||
t.Fatalf("expected report file: %v", err)
|
||||
}
|
||||
logData, _ := os.ReadFile(filepath.Join(dir, ".relspec", "lint.log"))
|
||||
if !strings.Contains(string(logData), "inspect:") {
|
||||
t.Fatalf("logfile missing inspect summary:\n%s", logData)
|
||||
}
|
||||
}
|
||||
|
||||
func TestJobRun_InspectJobFailsOnRuleError(t *testing.T) {
|
||||
dir := jobFixture(t, `version: 1
|
||||
jobs:
|
||||
lint:
|
||||
command: inspect
|
||||
inputs:
|
||||
- path: schema/core.dbml
|
||||
format: dbml
|
||||
rules: rules.yaml
|
||||
report:
|
||||
format: json
|
||||
path: build/report.json
|
||||
overwrite: true
|
||||
logfile: .relspec/lint.log
|
||||
`)
|
||||
// A rule set to "error" level for a violation the fixture triggers.
|
||||
writeFile(t, filepath.Join(dir, "rules.yaml"), `version: "1.0"
|
||||
rules:
|
||||
primary_key_naming:
|
||||
enabled: enforce
|
||||
function: primary_key_naming
|
||||
pattern: "^id_"
|
||||
message: "Primary key columns should start with id_"
|
||||
`)
|
||||
set := mustLoadSet(t, filepath.Join(dir, "relspec.yml"))
|
||||
err := executeJobPlan(set, "lint", false, false, &bytes.Buffer{})
|
||||
if err == nil || !strings.Contains(err.Error(), "error(s)") {
|
||||
t.Fatalf("expected inspect job to fail on rule error, got %v", err)
|
||||
}
|
||||
logData, _ := os.ReadFile(filepath.Join(dir, ".relspec", "lint.log"))
|
||||
if !strings.Contains(string(logData), "FAILED") {
|
||||
t.Fatalf("failed inspect job should log FAILED:\n%s", logData)
|
||||
}
|
||||
}
|
||||
|
||||
func TestJobRun_DiffJob(t *testing.T) {
|
||||
dir := jobFixture(t, `version: 1
|
||||
jobs:
|
||||
compare:
|
||||
command: diff
|
||||
inputs:
|
||||
- path: schema/core.dbml
|
||||
format: dbml
|
||||
- path: schema/tenant.dbml
|
||||
format: dbml
|
||||
report:
|
||||
format: json
|
||||
path: build/diff.json
|
||||
overwrite: true
|
||||
`)
|
||||
set := mustLoadSet(t, filepath.Join(dir, "relspec.yml"))
|
||||
if err := executeJobPlan(set, "compare", false, false, &bytes.Buffer{}); err != nil {
|
||||
t.Fatalf("execute diff job: %v", err)
|
||||
}
|
||||
out, err := os.ReadFile(filepath.Join(dir, "build", "diff.json"))
|
||||
if err != nil {
|
||||
t.Fatalf("read diff report: %v", err)
|
||||
}
|
||||
if len(out) == 0 {
|
||||
t.Fatal("diff report is empty")
|
||||
}
|
||||
}
|
||||
|
||||
func TestJobRun_FromJobWiring(t *testing.T) {
|
||||
dir := jobFixture(t, `version: 1
|
||||
jobs:
|
||||
a:
|
||||
command: convert
|
||||
inputs:
|
||||
- path: schema/core.dbml
|
||||
format: dbml
|
||||
output:
|
||||
format: json
|
||||
path: build/a.json
|
||||
overwrite: true
|
||||
b:
|
||||
command: convert
|
||||
inputs:
|
||||
- from_job: a
|
||||
output:
|
||||
format: yaml
|
||||
path: build/b.yaml
|
||||
overwrite: true
|
||||
`)
|
||||
set := mustLoadSet(t, filepath.Join(dir, "relspec.yml"))
|
||||
if err := executeJobPlan(set, "b", false, false, &bytes.Buffer{}); err != nil {
|
||||
t.Fatalf("execute from_job chain: %v", err)
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(dir, "build", "a.json")); err != nil {
|
||||
t.Fatalf("producer output missing: %v", err)
|
||||
}
|
||||
out, err := os.ReadFile(filepath.Join(dir, "build", "b.yaml"))
|
||||
if err != nil {
|
||||
t.Fatalf("consumer output missing: %v", err)
|
||||
}
|
||||
if !strings.Contains(string(out), "users") {
|
||||
t.Fatalf("consumer did not consume producer output:\n%s", out)
|
||||
}
|
||||
}
|
||||
|
||||
func TestJobRun_LogRotation(t *testing.T) {
|
||||
dir := jobFixture(t, `version: 1
|
||||
jobs:
|
||||
build:
|
||||
command: scripts-list
|
||||
script_dirs: [migrations]
|
||||
log_max_size: "150B"
|
||||
log_keep: 2
|
||||
logfile: .relspec/build.log
|
||||
`)
|
||||
writeFile(t, filepath.Join(dir, "migrations", "1_001_a.sql"), "CREATE TABLE a();\n")
|
||||
logPath := filepath.Join(dir, ".relspec", "build.log")
|
||||
writeFile(t, logPath, strings.Repeat("x", 300)+"\n")
|
||||
|
||||
set := mustLoadSet(t, filepath.Join(dir, "relspec.yml"))
|
||||
if err := executeJobPlan(set, "build", false, false, &bytes.Buffer{}); err != nil {
|
||||
t.Fatalf("execute job: %v", err)
|
||||
}
|
||||
rotated, err := os.ReadFile(logPath + ".1")
|
||||
if err != nil {
|
||||
t.Fatalf("expected rotated logfile build.log.1: %v", err)
|
||||
}
|
||||
if !strings.Contains(string(rotated), strings.Repeat("x", 300)) {
|
||||
t.Fatalf("rotated logfile should hold the old content")
|
||||
}
|
||||
fresh, err := os.ReadFile(logPath)
|
||||
if err != nil {
|
||||
t.Fatalf("expected fresh logfile: %v", err)
|
||||
}
|
||||
if strings.Contains(string(fresh), strings.Repeat("x", 300)) {
|
||||
t.Fatalf("fresh logfile should not contain the rotated-out content:\n%s", fresh)
|
||||
}
|
||||
if !strings.Contains(string(fresh), "OK") {
|
||||
t.Fatalf("fresh logfile should hold the new run:\n%s", fresh)
|
||||
}
|
||||
}
|
||||
|
||||
func TestJobRun_AtomicOutputLeavesOriginalOnFailure(t *testing.T) {
|
||||
dir := jobFixture(t, `version: 1
|
||||
jobs:
|
||||
x:
|
||||
command: convert
|
||||
inputs:
|
||||
- path: schema/core.dbml
|
||||
format: dbml
|
||||
output:
|
||||
format: json
|
||||
path: build/out.json
|
||||
overwrite: true
|
||||
`)
|
||||
// Seed the destination, then make its parent directory read-only so the
|
||||
// rename step fails. The seeded file must survive intact.
|
||||
seeded := filepath.Join(dir, "build", "out.json")
|
||||
writeFile(t, seeded, `{"seeded":true}`)
|
||||
if err := os.Chmod(filepath.Join(dir, "build"), 0o500); err != nil {
|
||||
t.Skipf("cannot chmod: %v", err)
|
||||
}
|
||||
t.Cleanup(func() { _ = os.Chmod(filepath.Join(dir, "build"), 0o755) })
|
||||
|
||||
set := mustLoadSet(t, filepath.Join(dir, "relspec.yml"))
|
||||
if err := executeJobPlan(set, "x", false, false, &bytes.Buffer{}); err == nil {
|
||||
t.Skip("write unexpectedly succeeded (running as root?)")
|
||||
}
|
||||
if err := os.Chmod(filepath.Join(dir, "build"), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
data, err := os.ReadFile(seeded)
|
||||
if err != nil {
|
||||
t.Fatalf("seeded file gone: %v", err)
|
||||
}
|
||||
if !strings.Contains(string(data), "seeded") {
|
||||
t.Fatalf("seeded file was corrupted: %s", data)
|
||||
}
|
||||
}
|
||||
|
||||
func TestJobRun_ScriptsExecMissingConnEnv(t *testing.T) {
|
||||
dir := jobFixture(t, `version: 1
|
||||
jobs:
|
||||
migrate:
|
||||
command: scripts-exec
|
||||
script_dirs: [migrations]
|
||||
output:
|
||||
conn_env: RELSPEC_TEST_EXEC_MISSING
|
||||
logfile: .relspec/migrate.log
|
||||
`)
|
||||
writeFile(t, filepath.Join(dir, "migrations", "1_001_a.sql"), "CREATE TABLE a();\n")
|
||||
os.Unsetenv("RELSPEC_TEST_EXEC_MISSING")
|
||||
set := mustLoadSet(t, filepath.Join(dir, "relspec.yml"))
|
||||
err := executeJobPlan(set, "migrate", false, false, &bytes.Buffer{})
|
||||
if err == nil || !strings.Contains(err.Error(), "conn_env") {
|
||||
t.Fatalf("expected missing conn_env error, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestJobRun_ScriptsExecDryRun(t *testing.T) {
|
||||
dir := jobFixture(t, `version: 1
|
||||
jobs:
|
||||
migrate:
|
||||
command: scripts-exec
|
||||
script_dirs: [migrations]
|
||||
output:
|
||||
conn_env: RELSPEC_TEST_EXEC_CONN
|
||||
`)
|
||||
writeFile(t, filepath.Join(dir, "migrations", "1_001_a.sql"), "CREATE TABLE a();\n")
|
||||
t.Setenv("RELSPEC_TEST_EXEC_CONN", "postgres://u:secretpw@h/db")
|
||||
set := mustLoadSet(t, filepath.Join(dir, "relspec.yml"))
|
||||
var buf bytes.Buffer
|
||||
if err := executeJobPlan(set, "migrate", true, false, &buf); err != nil {
|
||||
t.Fatalf("dry run: %v", err)
|
||||
}
|
||||
if strings.Contains(buf.String(), "secretpw") {
|
||||
t.Fatalf("plan leaked secret:\n%s", buf.String())
|
||||
}
|
||||
if !strings.Contains(buf.String(), "env:RELSPEC_TEST_EXEC_CONN") {
|
||||
t.Fatalf("plan should name the env var:\n%s", buf.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestJobRun_TemplDatabaseMode(t *testing.T) {
|
||||
dir := jobFixture(t, `version: 1
|
||||
jobs:
|
||||
docs:
|
||||
command: templ
|
||||
inputs:
|
||||
- path: schema/core.dbml
|
||||
format: dbml
|
||||
template: templates/schema.tmpl
|
||||
output:
|
||||
format: text
|
||||
path: build/schema.txt
|
||||
overwrite: true
|
||||
`)
|
||||
writeFile(t, filepath.Join(dir, "templates", "schema.tmpl"), "{{range .Database.Schemas}}{{range .Tables}}{{.Name}} {{end}}{{end}}")
|
||||
set := mustLoadSet(t, filepath.Join(dir, "relspec.yml"))
|
||||
if err := executeJobPlan(set, "docs", false, false, &bytes.Buffer{}); err != nil {
|
||||
t.Fatalf("execute templ job: %v", err)
|
||||
}
|
||||
out, err := os.ReadFile(filepath.Join(dir, "build", "schema.txt"))
|
||||
if err != nil {
|
||||
t.Fatalf("read templ output: %v", err)
|
||||
}
|
||||
if !strings.Contains(string(out), "users") {
|
||||
t.Fatalf("templ output missing users table: %s", out)
|
||||
}
|
||||
}
|
||||
|
||||
func TestJobRun_BunSqlTypesOption(t *testing.T) {
|
||||
dir := jobFixture(t, `version: 1
|
||||
jobs:
|
||||
models:
|
||||
command: convert
|
||||
inputs:
|
||||
- path: schema/core.dbml
|
||||
format: dbml
|
||||
output:
|
||||
format: bun
|
||||
path: build/models
|
||||
options:
|
||||
package: models
|
||||
types: sqltypes
|
||||
`)
|
||||
set := mustLoadSet(t, filepath.Join(dir, "relspec.yml"))
|
||||
if err := executeJobPlan(set, "models", false, false, &bytes.Buffer{}); err != nil {
|
||||
t.Fatalf("execute Bun job: %v", err)
|
||||
}
|
||||
generated, err := os.ReadFile(filepath.Join(dir, "build", "models"))
|
||||
if err != nil {
|
||||
t.Fatalf("read Bun output: %v", err)
|
||||
}
|
||||
if !strings.Contains(string(generated), "git.warky.dev/wdevs/relspecgo/pkg/sqltypes") {
|
||||
t.Fatalf("Bun output did not preserve options.types: %s", generated)
|
||||
}
|
||||
}
|
||||
|
||||
func TestJobValidation_TemplTextFormat(t *testing.T) {
|
||||
validDir := jobFixture(t, `version: 1
|
||||
jobs:
|
||||
docs:
|
||||
command: templ
|
||||
inputs: [{path: schema/core.dbml, format: dbml}]
|
||||
template: schema.tmpl
|
||||
output: {format: text, path: build/docs.txt}
|
||||
`)
|
||||
valid := mustLoadSet(t, filepath.Join(validDir, "relspec.yml"))
|
||||
if err := valid.Validate(); err != nil {
|
||||
t.Fatalf("text output should be valid for templ: %v", err)
|
||||
}
|
||||
invalidDir := jobFixture(t, `version: 1
|
||||
jobs:
|
||||
docs:
|
||||
command: templ
|
||||
inputs: [{path: schema/core.dbml, format: dbml}]
|
||||
template: schema.tmpl
|
||||
output: {format: json, path: build/docs.txt}
|
||||
`)
|
||||
invalid, err := jobs.Load([]string{filepath.Join(invalidDir, "relspec.yml")})
|
||||
if err != nil {
|
||||
t.Fatalf("load invalid manifest: %v", err)
|
||||
}
|
||||
if err := invalid.Validate(); err == nil || !strings.Contains(err.Error(), "output.format: text") {
|
||||
t.Fatalf("expected templ format rejection, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
+35
-2
@@ -5,10 +5,43 @@ import (
|
||||
"os"
|
||||
)
|
||||
|
||||
// asciiLogo (see version.go) is printed by the `version` command.
|
||||
|
||||
func main() {
|
||||
printVersionHeader(os.Args[1:])
|
||||
if err := rootCmd.Execute(); err != nil {
|
||||
args := os.Args[1:]
|
||||
isSilent := hasSilentFlag(args)
|
||||
if !isSilent {
|
||||
printVersionHeader(args)
|
||||
}
|
||||
|
||||
previousStderr := os.Stderr
|
||||
var nullOutput *os.File
|
||||
if isSilent {
|
||||
var err error
|
||||
nullOutput, err = os.OpenFile(os.DevNull, os.O_WRONLY, 0)
|
||||
if err != nil {
|
||||
fmt.Fprintln(previousStderr, err)
|
||||
os.Exit(1)
|
||||
}
|
||||
os.Stderr = nullOutput
|
||||
}
|
||||
|
||||
err := rootCmd.Execute()
|
||||
if nullOutput != nil {
|
||||
os.Stderr = previousStderr
|
||||
nullOutput.Close()
|
||||
}
|
||||
if err != nil {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
func hasSilentFlag(args []string) bool {
|
||||
for _, arg := range args {
|
||||
if arg == "--silent" || arg == "--silent=true" {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
+45
-5
@@ -59,7 +59,9 @@ var (
|
||||
mergeSkipTables string // Comma-separated table names to skip
|
||||
mergeVerbose bool
|
||||
mergeReportPath string // Path to write merge report
|
||||
mergeFullDDL bool // Execute full DDL instead of diffing the live pgsql database
|
||||
mergeFlattenSchema bool
|
||||
mergeDryRun bool
|
||||
)
|
||||
|
||||
var mergeCmd = &cobra.Command{
|
||||
@@ -127,7 +129,9 @@ func init() {
|
||||
mergeCmd.Flags().BoolVar(&mergeSkipSequences, "skip-sequences", false, "Skip sequences during merge")
|
||||
mergeCmd.Flags().StringVar(&mergeSkipTables, "skip-tables", "", "Comma-separated list of table names to skip during merge")
|
||||
mergeCmd.Flags().BoolVar(&mergeVerbose, "verbose", false, "Show verbose output")
|
||||
mergeCmd.Flags().BoolVar(&mergeFullDDL, "full-ddl", false, "pgsql database output: execute the full DDL instead of only the differences from the live database")
|
||||
mergeCmd.Flags().StringVar(&mergeReportPath, "merge-report", "", "Path to write merge report (JSON format)")
|
||||
mergeCmd.Flags().BoolVar(&mergeDryRun, "dry-run", false, "Read and merge in memory, then print the merge plan without writing any output")
|
||||
mergeCmd.Flags().BoolVar(&mergeFlattenSchema, "flatten-schema", false, "Flatten schema.table names to schema_table (useful for databases like SQLite that do not support schemas)")
|
||||
}
|
||||
|
||||
@@ -158,9 +162,7 @@ func runMerge(cmd *cobra.Command, args []string) error {
|
||||
}
|
||||
mergeTargetPath = expandPath(mergeTargetPath)
|
||||
} else if mergeTargetConn == "" {
|
||||
|
||||
return fmt.Errorf("--target-conn is required for pgsql format")
|
||||
|
||||
}
|
||||
|
||||
if mergeSourceType != "pgsql" {
|
||||
@@ -180,7 +182,7 @@ func runMerge(cmd *cobra.Command, args []string) error {
|
||||
}
|
||||
|
||||
// Step 1: Read target database
|
||||
fmt.Fprintf(os.Stderr, "[1/3] Reading target database...\n")
|
||||
fmt.Fprintf(os.Stderr, "[1/4] Reading target database...\n")
|
||||
fmt.Fprintf(os.Stderr, " Format: %s\n", mergeTargetType)
|
||||
if mergeTargetPath != "" {
|
||||
fmt.Fprintf(os.Stderr, " Path: %s\n", mergeTargetPath)
|
||||
@@ -197,7 +199,7 @@ func runMerge(cmd *cobra.Command, args []string) error {
|
||||
printDatabaseStats(targetDB)
|
||||
|
||||
// Step 2: Read source database(s)
|
||||
fmt.Fprintf(os.Stderr, "\n[2/3] Reading source database...\n")
|
||||
fmt.Fprintf(os.Stderr, "\n[2/4] Reading source database...\n")
|
||||
fmt.Fprintf(os.Stderr, " Format: %s\n", mergeSourceType)
|
||||
|
||||
var sourceDB *models.Database
|
||||
@@ -231,7 +233,7 @@ func runMerge(cmd *cobra.Command, args []string) error {
|
||||
printDatabaseStats(sourceDB)
|
||||
|
||||
// Step 3: Merge databases
|
||||
fmt.Fprintf(os.Stderr, "\n[3/3] Merging databases...\n")
|
||||
fmt.Fprintf(os.Stderr, "\n[3/4] Merging databases...\n")
|
||||
|
||||
opts := &merge.MergeOptions{
|
||||
SkipDomains: mergeSkipDomains,
|
||||
@@ -250,6 +252,7 @@ func runMerge(cmd *cobra.Command, args []string) error {
|
||||
}
|
||||
|
||||
result := merge.MergeDatabases(targetDB, sourceDB, opts)
|
||||
finalizeCommentedRefs(targetDB, stderrWarn)
|
||||
|
||||
// Update timestamp
|
||||
targetDB.UpdateDate()
|
||||
@@ -263,12 +266,38 @@ func runMerge(cmd *cobra.Command, args []string) error {
|
||||
merge.GetColumnTypeConflictSummary(result, 10))
|
||||
}
|
||||
|
||||
if mergeDryRun {
|
||||
if !isMergeOutputFormat(mergeOutputType) {
|
||||
return fmt.Errorf("dry run validation failed: Output: unsupported format '%s'", mergeOutputType)
|
||||
}
|
||||
out := outWriter(cmd)
|
||||
fmt.Fprintf(out, "RelSpec merge plan (dry run - nothing written):\n")
|
||||
fmt.Fprintf(out, " Target: %s database '%s'\n", mergeTargetType, targetDB.Name)
|
||||
fmt.Fprintf(out, " Source: %s database '%s'\n", mergeSourceType, sourceDB.Name)
|
||||
switch {
|
||||
case mergeOutputPath != "":
|
||||
fmt.Fprintf(out, " Output: %s -> %s\n", mergeOutputType, mergeOutputPath)
|
||||
case mergeOutputConn != "":
|
||||
fmt.Fprintf(out, " Output: %s -> %s\n", mergeOutputType, maskPassword(mergeOutputConn))
|
||||
default:
|
||||
fmt.Fprintf(out, " Output: %s\n", mergeOutputType)
|
||||
}
|
||||
fmt.Fprintf(out, " Result:\n")
|
||||
printDryRunPlan(out, targetDB)
|
||||
fmt.Fprintf(out, "\n%s\n", merge.GetMergeSummary(result))
|
||||
fmt.Fprintf(os.Stderr, "\n=== Dry run complete: no output written ===\n")
|
||||
return nil
|
||||
}
|
||||
|
||||
// Step 4: Write output
|
||||
fmt.Fprintf(os.Stderr, "\n[4/4] Writing output...\n")
|
||||
fmt.Fprintf(os.Stderr, " Format: %s\n", mergeOutputType)
|
||||
if mergeOutputPath != "" {
|
||||
fmt.Fprintf(os.Stderr, " Path: %s\n", mergeOutputPath)
|
||||
}
|
||||
if mergeOutputConn != "" {
|
||||
fmt.Fprintf(os.Stderr, " Conn: %s\n", maskPassword(mergeOutputConn))
|
||||
}
|
||||
|
||||
err = writeDatabaseForMerge(mergeOutputType, mergeOutputPath, mergeOutputConn, targetDB, "Output", mergeFlattenSchema)
|
||||
if err != nil {
|
||||
@@ -281,6 +310,16 @@ func runMerge(cmd *cobra.Command, args []string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// isMergeOutputFormat reports whether writeDatabaseForMerge supports dbType.
|
||||
func isMergeOutputFormat(dbType string) bool {
|
||||
switch strings.ToLower(dbType) {
|
||||
case "dbml", "dctx", "drawdb", "graphql", "json", "yaml", "gorm", "bun",
|
||||
"drizzle", "prisma", "typeorm", "sqlite", "sqlite3", "pgsql":
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func readDatabaseForMerge(dbType, filePath, connString, label string) (*models.Database, error) {
|
||||
var reader readers.Reader
|
||||
|
||||
@@ -441,6 +480,7 @@ func writeDatabaseForMerge(dbType, filePath, connString string, db *models.Datab
|
||||
if connString != "" {
|
||||
writerOpts.Metadata = map[string]interface{}{
|
||||
"connection_string": connString,
|
||||
"full_ddl": mergeFullDDL,
|
||||
}
|
||||
// Add report path if merge report is enabled
|
||||
if mergeReportPath != "" {
|
||||
|
||||
@@ -105,7 +105,7 @@ func TestRunMerge_FromListPathWithSpaces(t *testing.T) {
|
||||
defer restoreMergeState(saved)
|
||||
|
||||
spacedDir := filepath.Join(t.TempDir(), "my schema files")
|
||||
if err := os.MkdirAll(spacedDir, 0755); err != nil {
|
||||
if err := os.MkdirAll(spacedDir, 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
targetFile := filepath.Join(spacedDir, "target schema.json")
|
||||
|
||||
@@ -1,6 +1,9 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
|
||||
"git.warky.dev/wdevs/relspecgo/pkg/readers"
|
||||
"git.warky.dev/wdevs/relspecgo/pkg/writers"
|
||||
)
|
||||
@@ -10,17 +13,23 @@ func newReaderOptions(filePath, connString string) *readers.ReaderOptions {
|
||||
FilePath: filePath,
|
||||
ConnectionString: connString,
|
||||
Prisma7: prisma7,
|
||||
StrictDirectives: strictDirectives,
|
||||
Progress: func(message string) {
|
||||
fmt.Fprintf(os.Stderr, " → %s\n", message)
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func newWriterOptions(outputPath, packageName string, flattenSchema bool, nullableTypes, nullableArrays string, continueOnError bool) *writers.WriterOptions {
|
||||
return &writers.WriterOptions{
|
||||
OutputPath: outputPath,
|
||||
PackageName: packageName,
|
||||
FlattenSchema: flattenSchema,
|
||||
NullableTypes: nullableTypes,
|
||||
NullableArrays: nullableArrays,
|
||||
Prisma7: prisma7,
|
||||
ContinueOnError: continueOnError,
|
||||
OutputPath: outputPath,
|
||||
PackageName: packageName,
|
||||
FlattenSchema: flattenSchema,
|
||||
NullableTypes: nullableTypes,
|
||||
NullableArrays: nullableArrays,
|
||||
TypeMappings: typeMappings,
|
||||
Prisma7: prisma7,
|
||||
ContinueOnError: continueOnError,
|
||||
StrictDirectives: strictDirectives,
|
||||
}
|
||||
}
|
||||
|
||||
+22
-37
@@ -2,50 +2,26 @@ package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"runtime/debug"
|
||||
"time"
|
||||
|
||||
"github.com/spf13/cobra"
|
||||
|
||||
"git.warky.dev/wdevs/relspecgo/pkg/buildinfo"
|
||||
"git.warky.dev/wdevs/relspecgo/pkg/writers"
|
||||
)
|
||||
|
||||
// version/buildDate mirror pkg/buildinfo so existing call sites keep working.
|
||||
// The actual values are set there via ldflags (see Makefile).
|
||||
var (
|
||||
// Version information, set via ldflags during build
|
||||
version = "dev"
|
||||
buildDate = "unknown"
|
||||
prisma7 bool
|
||||
noVersion bool
|
||||
version = buildinfo.Version
|
||||
buildDate = buildinfo.BuildDate
|
||||
prisma7 bool
|
||||
noVersion bool
|
||||
silent bool
|
||||
strictDirectives bool
|
||||
typeMapFlags []string
|
||||
typeMappings map[string]string
|
||||
)
|
||||
|
||||
func init() {
|
||||
// If version wasn't set via ldflags, try to get it from build info
|
||||
if version == "dev" {
|
||||
if info, ok := debug.ReadBuildInfo(); ok {
|
||||
// Try to get version from VCS
|
||||
var vcsRevision, vcsTime string
|
||||
for _, setting := range info.Settings {
|
||||
switch setting.Key {
|
||||
case "vcs.revision":
|
||||
if len(setting.Value) >= 7 {
|
||||
vcsRevision = setting.Value[:7]
|
||||
}
|
||||
case "vcs.time":
|
||||
vcsTime = setting.Value
|
||||
}
|
||||
}
|
||||
|
||||
if vcsRevision != "" {
|
||||
version = vcsRevision
|
||||
}
|
||||
|
||||
if vcsTime != "" {
|
||||
if t, err := time.Parse(time.RFC3339, vcsTime); err == nil {
|
||||
buildDate = t.UTC().Format("2006-01-02 15:04:05 UTC")
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
var rootCmd = &cobra.Command{
|
||||
Use: "relspec",
|
||||
Short: "RelSpec - Database schema conversion and analysis tool",
|
||||
@@ -55,10 +31,16 @@ bidirectional conversion between various database schema formats.
|
||||
It reads database schemas from multiple sources (live databases, DBML,
|
||||
DCTX, DrawDB, etc.) and writes them to various formats (GORM, Bun,
|
||||
JSON, YAML, SQL, etc.).`,
|
||||
PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
|
||||
var err error
|
||||
typeMappings, err = writers.ParseTypeMappings(typeMapFlags)
|
||||
return err
|
||||
},
|
||||
}
|
||||
|
||||
func init() {
|
||||
rootCmd.AddCommand(convertCmd)
|
||||
rootCmd.AddCommand(batchCmd)
|
||||
rootCmd.AddCommand(diffCmd)
|
||||
rootCmd.AddCommand(inspectCmd)
|
||||
rootCmd.AddCommand(scriptsCmd)
|
||||
@@ -71,7 +53,10 @@ func init() {
|
||||
rootCmd.AddCommand(versionCmd)
|
||||
rootCmd.AddCommand(reportCmd)
|
||||
rootCmd.PersistentFlags().BoolVar(&prisma7, "prisma7", false, "Use Prisma 7 generator conventions when reading/writing Prisma schemas")
|
||||
rootCmd.PersistentFlags().StringArrayVar(&typeMapFlags, "type-map", nil, "Override a SQL-to-Go type mapping for bun/gorm output as sqltype=gotype (repeatable), e.g. --type-map uuid=uuid.UUID --type-map numeric=decimal.Decimal")
|
||||
rootCmd.PersistentFlags().BoolVar(&noVersion, "no-version", false, "Suppress the RelSpec version header")
|
||||
rootCmd.PersistentFlags().BoolVar(&silent, "silent", false, "Suppress progress and status messages (errors are still shown)")
|
||||
rootCmd.PersistentFlags().BoolVar(&strictDirectives, "strict-directives", false, "Fail on unknown or untranslatable DBML dialect directives (@postgres:, @sqlite:, …)")
|
||||
}
|
||||
|
||||
// printVersionHeader prints the "RelSpec <version> (built: <date>)" banner
|
||||
|
||||
+73
-6
@@ -24,6 +24,7 @@ var (
|
||||
splitExcludeTables string
|
||||
splitNullableTypes string
|
||||
splitNullableArrays string
|
||||
splitDryRun bool
|
||||
)
|
||||
|
||||
var splitCmd = &cobra.Command{
|
||||
@@ -115,6 +116,8 @@ func init() {
|
||||
splitCmd.Flags().StringVar(&splitNullableTypes, "types", "", "Nullable type package for code-gen writers (bun/gorm): 'baselib' (default, Go pointer types), 'stdlib' (database/sql), or 'sqltypes'")
|
||||
splitCmd.Flags().StringVar(&splitNullableArrays, "array-nullable", "", "Nullable PostgreSQL array representation for the Bun writer in stdlib/baselib --types mode: 'slice' (default, plain slice) or 'pointer_slice' (*[]T, distinguishes NULL from '{}')")
|
||||
|
||||
splitCmd.Flags().BoolVar(&splitDryRun, "dry-run", false, "Read, filter and validate the selection and print the plan without writing any output")
|
||||
|
||||
err := splitCmd.MarkFlagRequired("from")
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "Error marking from flag as required: %v\n", err)
|
||||
@@ -174,6 +177,26 @@ func runSplit(cmd *cobra.Command, args []string) error {
|
||||
}
|
||||
fmt.Fprintf(os.Stderr, " ✓ Filtered to: %d schema(s), %d table(s)\n\n", len(filteredDB.Schemas), filteredTables)
|
||||
|
||||
if splitDryRun {
|
||||
if err := validateWriteTarget(filteredDB, splitTargetType, splitPackageName, "", ""); err != nil {
|
||||
return fmt.Errorf("dry run validation failed: %w", err)
|
||||
}
|
||||
out := outWriter(cmd)
|
||||
fmt.Fprintf(out, "RelSpec split plan (dry run - nothing written):\n")
|
||||
fmt.Fprintf(out, " Input: %s database '%s'\n", splitSourceType, db.Name)
|
||||
fmt.Fprintf(out, " Output: %s -> %s\n", splitTargetType, splitTargetPath)
|
||||
fmt.Fprintf(out, " Selection: %s\n", splitSelection{
|
||||
Schemas: parseCommaSeparated(splitSchemas),
|
||||
Tables: parseCommaSeparated(splitTables),
|
||||
ExcludeSchemas: parseCommaSeparated(splitExcludeSchema),
|
||||
ExcludeTables: parseCommaSeparated(splitExcludeTables),
|
||||
DatabaseName: splitDatabaseName,
|
||||
}.summary())
|
||||
printDryRunPlan(out, filteredDB)
|
||||
fmt.Fprintf(os.Stderr, "=== Dry run complete: no output written ===\n\n")
|
||||
return nil
|
||||
}
|
||||
|
||||
// Write to target format
|
||||
fmt.Fprintf(os.Stderr, "[3/3] Writing to target format...\n")
|
||||
fmt.Fprintf(os.Stderr, " Format: %s\n", splitTargetType)
|
||||
@@ -205,8 +228,52 @@ func runSplit(cmd *cobra.Command, args []string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// filterDatabase filters the database based on provided criteria
|
||||
// splitSelection is the schema/table selection for a split, independent of the
|
||||
// CLI flag globals so the job runner can build one directly.
|
||||
type splitSelection struct {
|
||||
Schemas []string
|
||||
Tables []string
|
||||
ExcludeSchemas []string
|
||||
ExcludeTables []string
|
||||
DatabaseName string
|
||||
}
|
||||
|
||||
// summary renders a one-line human description of the selection.
|
||||
func (s splitSelection) summary() string {
|
||||
var parts []string
|
||||
if len(s.Schemas) > 0 {
|
||||
parts = append(parts, "schemas="+strings.Join(s.Schemas, ","))
|
||||
}
|
||||
if len(s.Tables) > 0 {
|
||||
parts = append(parts, "tables="+strings.Join(s.Tables, ","))
|
||||
}
|
||||
if len(s.ExcludeSchemas) > 0 {
|
||||
parts = append(parts, "exclude_schemas="+strings.Join(s.ExcludeSchemas, ","))
|
||||
}
|
||||
if len(s.ExcludeTables) > 0 {
|
||||
parts = append(parts, "exclude_tables="+strings.Join(s.ExcludeTables, ","))
|
||||
}
|
||||
if s.DatabaseName != "" {
|
||||
parts = append(parts, "database_name="+s.DatabaseName)
|
||||
}
|
||||
if len(parts) == 0 {
|
||||
return "(all schemas/tables)"
|
||||
}
|
||||
return strings.Join(parts, " ")
|
||||
}
|
||||
|
||||
// filterDatabase filters the database based on the CLI split flags.
|
||||
func filterDatabase(db *models.Database) (*models.Database, error) {
|
||||
return filterDatabaseSelection(db, splitSelection{
|
||||
Schemas: parseCommaSeparated(splitSchemas),
|
||||
Tables: parseCommaSeparated(splitTables),
|
||||
ExcludeSchemas: parseCommaSeparated(splitExcludeSchema),
|
||||
ExcludeTables: parseCommaSeparated(splitExcludeTables),
|
||||
})
|
||||
}
|
||||
|
||||
// filterDatabaseSelection filters db down to the schemas/tables named by sel.
|
||||
func filterDatabaseSelection(db *models.Database, sel splitSelection) (*models.Database, error) {
|
||||
filteredDB := &models.Database{
|
||||
Name: db.Name,
|
||||
Description: db.Description,
|
||||
@@ -220,11 +287,11 @@ func filterDatabase(db *models.Database) (*models.Database, error) {
|
||||
Domains: db.Domains, // Keep domains for now
|
||||
}
|
||||
|
||||
// Parse filter flags
|
||||
includeSchemas := parseCommaSeparated(splitSchemas)
|
||||
includeTables := parseCommaSeparated(splitTables)
|
||||
excludeSchemas := parseCommaSeparated(splitExcludeSchema)
|
||||
excludeTables := parseCommaSeparated(splitExcludeTables)
|
||||
// Selection criteria
|
||||
includeSchemas := sel.Schemas
|
||||
includeTables := sel.Tables
|
||||
excludeSchemas := sel.ExcludeSchemas
|
||||
excludeTables := sel.ExcludeTables
|
||||
|
||||
// Convert table names to lowercase for case-insensitive matching
|
||||
includeTablesLower := make(map[string]bool)
|
||||
|
||||
@@ -114,6 +114,7 @@ func runTempl(cmd *cobra.Command, args []string) error {
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to read source: %w", err)
|
||||
}
|
||||
finalizeCommentedRefs(db, stderrWarn)
|
||||
|
||||
// Print database stats
|
||||
schemaCount := len(db.Schemas)
|
||||
|
||||
@@ -10,7 +10,7 @@ import (
|
||||
func writeTestTemplate(t *testing.T, path string) {
|
||||
t.Helper()
|
||||
content := []byte(`{{.Name}}`)
|
||||
if err := os.WriteFile(path, content, 0644); err != nil {
|
||||
if err := os.WriteFile(path, content, 0o644); err != nil {
|
||||
t.Fatalf("failed to write template file %s: %v", path, err)
|
||||
}
|
||||
}
|
||||
@@ -104,7 +104,7 @@ func TestRunTempl_FromListPathWithSpaces(t *testing.T) {
|
||||
defer restoreTemplState(saved)
|
||||
|
||||
spacedDir := filepath.Join(t.TempDir(), "my schema files")
|
||||
if err := os.MkdirAll(spacedDir, 0755); err != nil {
|
||||
if err := os.MkdirAll(spacedDir, 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
file1 := filepath.Join(spacedDir, "users schema.json")
|
||||
|
||||
@@ -66,7 +66,7 @@ func writeTestJSON(t *testing.T, path string, tableNames []string) {
|
||||
if err != nil {
|
||||
t.Fatalf("failed to marshal test JSON: %v", err)
|
||||
}
|
||||
if err := os.WriteFile(path, data, 0644); err != nil {
|
||||
if err := os.WriteFile(path, data, 0o644); err != nil {
|
||||
t.Fatalf("failed to write test file %s: %v", path, err)
|
||||
}
|
||||
}
|
||||
@@ -100,7 +100,7 @@ func writeTestJSONWithSingleColumnType(t *testing.T, path, tableName, columnType
|
||||
if err != nil {
|
||||
t.Fatalf("failed to marshal test JSON: %v", err)
|
||||
}
|
||||
if err := os.WriteFile(path, data, 0644); err != nil {
|
||||
if err := os.WriteFile(path, data, 0o644); err != nil {
|
||||
t.Fatalf("failed to write test file %s: %v", path, err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -4,13 +4,16 @@ import (
|
||||
"fmt"
|
||||
|
||||
"github.com/spf13/cobra"
|
||||
|
||||
"git.warky.dev/wdevs/relspecgo/pkg/buildinfo"
|
||||
)
|
||||
|
||||
var versionCmd = &cobra.Command{
|
||||
Use: "version",
|
||||
Short: "Print version information",
|
||||
Run: func(cmd *cobra.Command, args []string) {
|
||||
fmt.Printf("RelSpec %s\n", version)
|
||||
fmt.Printf("Built: %s\n", buildDate)
|
||||
fmt.Print(buildinfo.AsciiLogo)
|
||||
fmt.Printf("RelSpec %s\n", buildinfo.Version)
|
||||
fmt.Printf("Built: %s\n", buildinfo.BuildDate)
|
||||
},
|
||||
}
|
||||
|
||||
@@ -0,0 +1,152 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/fs"
|
||||
"os"
|
||||
"os/signal"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"syscall"
|
||||
"time"
|
||||
)
|
||||
|
||||
// watchSnapshot maps a file path to its modification time and size.
|
||||
type watchSnapshot map[string]string
|
||||
|
||||
// takeWatchSnapshot records the state of every file under the given paths.
|
||||
// Directories are walked recursively. Anything at or below the excluded path
|
||||
// (typically the output path) is skipped so regenerating output does not
|
||||
// retrigger the watcher. Missing paths are simply absent from the snapshot, so
|
||||
// creating them later counts as a change.
|
||||
func takeWatchSnapshot(paths []string, exclude string) watchSnapshot {
|
||||
snap := watchSnapshot{}
|
||||
exclude = absPathOrSelf(exclude)
|
||||
for _, root := range paths {
|
||||
_ = filepath.WalkDir(root, func(p string, d fs.DirEntry, err error) error {
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
if exclude != "" && isWithin(absPathOrSelf(p), exclude) {
|
||||
if d.IsDir() {
|
||||
return filepath.SkipDir
|
||||
}
|
||||
return nil
|
||||
}
|
||||
if d.IsDir() {
|
||||
return nil
|
||||
}
|
||||
info, err := d.Info()
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
snap[p] = fmt.Sprintf("%d-%d", info.ModTime().UnixNano(), info.Size())
|
||||
return nil
|
||||
})
|
||||
}
|
||||
return snap
|
||||
}
|
||||
|
||||
func (s watchSnapshot) equal(o watchSnapshot) bool {
|
||||
if len(s) != len(o) {
|
||||
return false
|
||||
}
|
||||
for k, v := range s {
|
||||
if o[k] != v {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func absPathOrSelf(p string) string {
|
||||
if p == "" {
|
||||
return ""
|
||||
}
|
||||
if abs, err := filepath.Abs(p); err == nil {
|
||||
return abs
|
||||
}
|
||||
return p
|
||||
}
|
||||
|
||||
// isWithin reports whether path equals dir or is located below it.
|
||||
func isWithin(path, dir string) bool {
|
||||
if path == dir {
|
||||
return true
|
||||
}
|
||||
return strings.HasPrefix(path, dir+string(filepath.Separator))
|
||||
}
|
||||
|
||||
// watchLoop runs fn once immediately and again whenever the watched paths
|
||||
// change, until ctx is cancelled. Changes are debounced: fn runs only after
|
||||
// the snapshot has stayed unchanged for one poll interval. Errors from fn are
|
||||
// reported to w and do not stop the loop.
|
||||
func watchLoop(ctx context.Context, w io.Writer, paths []string, exclude string, interval time.Duration, fn func() error) {
|
||||
run := func() {
|
||||
if err := fn(); err != nil {
|
||||
fmt.Fprintf(w, "Error: %v\n", err)
|
||||
}
|
||||
fmt.Fprintf(w, "Watching for changes (Ctrl-C to stop)...\n")
|
||||
}
|
||||
|
||||
last := takeWatchSnapshot(paths, exclude)
|
||||
run()
|
||||
|
||||
ticker := time.NewTicker(interval)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-ticker.C:
|
||||
}
|
||||
cur := takeWatchSnapshot(paths, exclude)
|
||||
if cur.equal(last) {
|
||||
continue
|
||||
}
|
||||
// Debounce: wait until writes settle.
|
||||
for settled := false; !settled; {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-time.After(interval):
|
||||
}
|
||||
next := takeWatchSnapshot(paths, exclude)
|
||||
settled = next.equal(cur)
|
||||
cur = next
|
||||
}
|
||||
fmt.Fprintf(w, "\nChange detected, regenerating...\n")
|
||||
last = cur
|
||||
run()
|
||||
}
|
||||
}
|
||||
|
||||
// runConvertWatch runs the conversion once and then again whenever the source
|
||||
// files change, until interrupted.
|
||||
func runConvertWatch(parent context.Context, w io.Writer, run func() error) error {
|
||||
var paths []string
|
||||
switch {
|
||||
case len(convertFromList) > 0:
|
||||
paths = convertFromList
|
||||
case convertSourcePath != "":
|
||||
paths = []string{convertSourcePath}
|
||||
default:
|
||||
return fmt.Errorf("--watch requires --from-path or --from-list (live database connections cannot be watched)")
|
||||
}
|
||||
if convertDryRun {
|
||||
return fmt.Errorf("--watch cannot be combined with --dry-run")
|
||||
}
|
||||
if convertWatchInterval <= 0 {
|
||||
return fmt.Errorf("--watch-interval must be positive")
|
||||
}
|
||||
if parent == nil {
|
||||
parent = context.Background()
|
||||
}
|
||||
|
||||
ctx, stop := signal.NotifyContext(parent, os.Interrupt, syscall.SIGTERM)
|
||||
defer stop()
|
||||
watchLoop(ctx, w, paths, convertTargetPath, convertWatchInterval, run)
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,92 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestWatchSnapshotExcludesOutput(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
out := filepath.Join(dir, "out")
|
||||
if err := os.MkdirAll(out, 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
src := filepath.Join(dir, "schema.dbml")
|
||||
if err := os.WriteFile(src, []byte("a"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
before := takeWatchSnapshot([]string{dir}, out)
|
||||
if err := os.WriteFile(filepath.Join(out, "gen.go"), []byte("x"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if after := takeWatchSnapshot([]string{dir}, out); !before.equal(after) {
|
||||
t.Errorf("writing into the excluded output path changed the snapshot")
|
||||
}
|
||||
if err := os.WriteFile(src, []byte("changed"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if after := takeWatchSnapshot([]string{dir}, out); before.equal(after) {
|
||||
t.Errorf("modifying a source file did not change the snapshot")
|
||||
}
|
||||
}
|
||||
|
||||
func TestWatchLoopRerunsOnChange(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
src := filepath.Join(dir, "schema.dbml")
|
||||
if err := os.WriteFile(src, []byte("a"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
var runs atomic.Int32
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
done := make(chan struct{})
|
||||
go func() {
|
||||
defer close(done)
|
||||
watchLoop(ctx, io.Discard, []string{src}, "", 10*time.Millisecond, func() error {
|
||||
runs.Add(1)
|
||||
return nil
|
||||
})
|
||||
}()
|
||||
|
||||
waitFor(t, func() bool { return runs.Load() == 1 })
|
||||
if err := os.WriteFile(src, []byte("changed content"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
waitFor(t, func() bool { return runs.Load() == 2 })
|
||||
cancel()
|
||||
<-done
|
||||
}
|
||||
|
||||
func TestRunConvertWatchValidation(t *testing.T) {
|
||||
oldPath, oldList, oldDry, oldInt := convertSourcePath, convertFromList, convertDryRun, convertWatchInterval
|
||||
defer func() {
|
||||
convertSourcePath, convertFromList, convertDryRun, convertWatchInterval = oldPath, oldList, oldDry, oldInt
|
||||
}()
|
||||
convertSourcePath, convertFromList, convertDryRun, convertWatchInterval = "", nil, false, time.Second
|
||||
if err := runConvertWatch(context.Background(), &bytes.Buffer{}, nil); err == nil {
|
||||
t.Error("expected error without --from-path/--from-list")
|
||||
}
|
||||
convertSourcePath, convertDryRun = "x.dbml", true
|
||||
if err := runConvertWatch(context.Background(), &bytes.Buffer{}, nil); err == nil {
|
||||
t.Error("expected error combining --watch with --dry-run")
|
||||
}
|
||||
}
|
||||
|
||||
func waitFor(t *testing.T, cond func() bool) {
|
||||
t.Helper()
|
||||
deadline := time.Now().Add(5 * time.Second)
|
||||
for time.Now().Before(deadline) {
|
||||
if cond() {
|
||||
return
|
||||
}
|
||||
time.Sleep(5 * time.Millisecond)
|
||||
}
|
||||
t.Fatal("condition not met in time")
|
||||
}
|
||||
@@ -0,0 +1,115 @@
|
||||
# DBML Dialect Directives
|
||||
|
||||
DBML has no dialect-neutral way to express database-specific features such as
|
||||
PostgreSQL table partitioning or SQLite `WITHOUT ROWID`. RelSpec adds **dialect
|
||||
directives** — explicit, parseable lines embedded in a `.dbml` file that are:
|
||||
|
||||
- stored losslessly in the intermediate model (under each object's `Metadata`),
|
||||
- preserved unchanged through a `DBML → model → DBML` round-trip,
|
||||
- translated to SQL **only** by the writer for the matching dialect
|
||||
(`@postgres:` clauses appear in PostgreSQL output, never in SQLite output, and
|
||||
vice-versa).
|
||||
|
||||
## Grammar
|
||||
|
||||
A directive is a single line, matched on its trimmed content:
|
||||
|
||||
```
|
||||
@<namespace>[(<target>)]: <args>
|
||||
```
|
||||
|
||||
| Part | Rules |
|
||||
|------|-------|
|
||||
| `namespace` | `^[a-z][a-z0-9_]*$` — e.g. `postgres`, `sqlite`. Future dialects allowed. |
|
||||
| `(target)` | Optional. A **column name** only, valid only on a directive line inside a table body. Bare or single/double quoted. |
|
||||
| `args` | Everything after the first `:`, trimmed. Otherwise preserved **verbatim**. Must be non-empty. |
|
||||
|
||||
The **key** of a directive is derived: the lowercased first whitespace-delimited
|
||||
token of `args` (`partition by RANGE (created_at)` → `partition`). It drives
|
||||
duplicate detection and writer dispatch.
|
||||
|
||||
## Location
|
||||
|
||||
Where the line appears determines which object it attaches to:
|
||||
|
||||
| Position in the file | Attaches to |
|
||||
|----------------------|-------------|
|
||||
| Before the first `Table {` | database (`db.Metadata`) |
|
||||
| Table body, no `(target)` | that table |
|
||||
| Table body, `(col)` target | column `col` of that table (error if `col` is unknown) |
|
||||
| Inside an `indexes { }` block | the **most recently listed** index entry in that block; `(target)` is not allowed |
|
||||
|
||||
```dbml
|
||||
@postgres: search_path myapp -- database
|
||||
|
||||
Table myapp.events {
|
||||
id bigint [pk]
|
||||
created_at timestamp [not null]
|
||||
@postgres(id): identity always -- column "id"
|
||||
@postgres: partition by RANGE (created_at) -- table
|
||||
@postgres: tablespace fast_data -- table
|
||||
@sqlite: without rowid -- table
|
||||
|
||||
indexes {
|
||||
(created_at) [name: 'idx_events_created']
|
||||
@postgres: with (fillfactor=90) -- index "idx_events_created"
|
||||
@postgres: tablespace idx_space -- index "idx_events_created"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## Duplicate policy
|
||||
|
||||
- **Repeatable by default** — every directive with the same `(namespace, key)` at
|
||||
one location is kept, in source order.
|
||||
- **Singletons** raise a line-numbered error on a second occurrence at the same
|
||||
location. Current singletons: `postgres` `partition`, `tablespace`, `inherits`,
|
||||
`storage`, `compression`, `identity`; `sqlite` `without`, `strict`, `collate`.
|
||||
|
||||
## Strict mode
|
||||
|
||||
CLI flag `--strict-directives` (also `ReaderOptions.StrictDirectives` /
|
||||
`WriterOptions.StrictDirectives`):
|
||||
|
||||
- **Reader**: an unknown namespace or key is a hard error. Without strict mode it
|
||||
is stored and preserved silently, and round-trips unchanged.
|
||||
- **PostgreSQL / SQLite writer**: a directive for **that** writer's own dialect
|
||||
whose key it cannot translate is a hard error. Without strict mode, translatable
|
||||
keys are emitted and the rest are skipped. Directives for other dialects are
|
||||
always ignored, never emitted.
|
||||
|
||||
## Errors
|
||||
|
||||
All are line-numbered (`dbml: line N: …`):
|
||||
|
||||
- no colon, or empty `args`
|
||||
- namespace empty or not matching `[a-z][a-z0-9_]*`
|
||||
- `(target)` naming an unknown column, or used at the top level / in an `indexes` block
|
||||
- a directive in the catalog used at a location it is not valid for
|
||||
- duplicate singleton at the same location
|
||||
- (strict mode) unknown `(namespace, key)`
|
||||
|
||||
## Supported directive matrix
|
||||
|
||||
### `@postgres`
|
||||
|
||||
| Key | Locations | SQL emitted | Notes |
|
||||
|-----|-----------|-------------|-------|
|
||||
| `partition` | table | `PARTITION BY <args>` appended to `CREATE TABLE` | e.g. `@postgres: partition by RANGE (created_at)` |
|
||||
| `inherits` | table | `INHERITS (<args>)` — args verbatim | |
|
||||
| `with` | table, index | `WITH (<params>)` | On an index, wins over `WITH` derived from the index comment. `@postgres: with (fillfactor=90)` |
|
||||
| `tablespace` | table, index | `TABLESPACE <name>` | Emitted after `WITH`, before `WHERE` on indexes |
|
||||
| `storage` | column | `STORAGE <mode>` in the column definition | e.g. `@postgres(blob): storage external` |
|
||||
| `compression` | column | `COMPRESSION <method>` | |
|
||||
| `identity` | column | `identity always` → `GENERATED ALWAYS AS IDENTITY`; `identity default` / `identity by default` → `GENERATED BY DEFAULT AS IDENTITY` | |
|
||||
|
||||
### `@sqlite`
|
||||
|
||||
| Key | Locations | SQL emitted | Notes |
|
||||
|-----|-----------|-------------|-------|
|
||||
| `without` | table | `WITHOUT ROWID` table option | `@sqlite: without rowid` |
|
||||
| `strict` | table | `STRICT` table option | `WITHOUT ROWID` is emitted before `STRICT` |
|
||||
| `collate` | column | ` COLLATE <name>` in the column definition | e.g. `@sqlite(name): collate NOCASE` |
|
||||
|
||||
Unknown namespaces and keys not in these tables are still preserved losslessly
|
||||
(and round-trip through the DBML writer) whenever strict mode is off.
|
||||
@@ -0,0 +1,103 @@
|
||||
# Format Usage Examples
|
||||
|
||||
Examples for `relspec convert` covering the file-based reader and writer
|
||||
formats. The "Writers" and "Readers" sections below were run against
|
||||
`examples/test_schema.dbml`. The cross-format and live-database examples were
|
||||
not run; they follow the flags shown in `relspec convert --help` and require
|
||||
matching input files or reachable databases.
|
||||
|
||||
Any reader can be combined with any writer: pick `--from`/`--from-path` for the
|
||||
source and `--to`/`--to-path` for the target. Add `--silent` to suppress progress
|
||||
output.
|
||||
|
||||
## Writers: DBML to every format
|
||||
|
||||
```bash
|
||||
S="--from dbml --from-path examples/test_schema.dbml"
|
||||
|
||||
relspec convert $S --to json --to-path schema.json
|
||||
relspec convert $S --to yaml --to-path schema.yaml
|
||||
relspec convert $S --to dctx --to-path schema.dctx
|
||||
relspec convert $S --to drawdb --to-path schema.drawdb.json
|
||||
relspec convert $S --to graphql --to-path schema.graphql
|
||||
relspec convert $S --to prisma --to-path schema.prisma
|
||||
relspec convert $S --to pgsql --to-path schema.pg.sql
|
||||
relspec convert $S --to mssql --to-path schema.mssql.sql
|
||||
relspec convert $S --to sqlite --to-path schema.sqlite.sql
|
||||
relspec convert $S --to drizzle --to-path schema.ts
|
||||
relspec convert $S --to typeorm --to-path entities.ts
|
||||
relspec convert $S --to gorm --to-path models.go --package models
|
||||
relspec convert $S --to bun --to-path models.go --package models
|
||||
```
|
||||
|
||||
Notes:
|
||||
|
||||
- Code-generation writers (`gorm`, `bun`) take `--package`. They also accept
|
||||
`--types baselib|stdlib|sqltypes` to choose the nullable type package.
|
||||
- When `--to-path` is a directory it must already exist.
|
||||
- `sqlite` output automatically flattens `schema.table` names. Use
|
||||
`--flatten-schema` for other formats if the target has no schema support.
|
||||
- `dctx` supports a single schema only; use `--schema <name>` to select one.
|
||||
|
||||
## Readers: file-based formats into DBML (or JSON where noted)
|
||||
|
||||
```bash
|
||||
relspec convert --from json --from-path schema.json --to dbml --to-path out.dbml
|
||||
relspec convert --from yaml --from-path schema.yaml --to dbml --to-path out.dbml
|
||||
relspec convert --from dctx --from-path schema.dctx --to dbml --to-path out.dbml
|
||||
relspec convert --from drawdb --from-path schema.drawdb.json --to dbml --to-path out.dbml
|
||||
relspec convert --from graphql --from-path schema.graphql --to dbml --to-path out.dbml
|
||||
relspec convert --from prisma --from-path schema.prisma --to dbml --to-path out.dbml
|
||||
relspec convert --from drizzle --from-path schema.ts --to dbml --to-path out.dbml
|
||||
relspec convert --from typeorm --from-path entities.ts --to dbml --to-path out.dbml
|
||||
relspec convert --from bun --from-path models.go --to dbml --to-path out.dbml
|
||||
relspec convert --from gorm --from-path models.go --to json --to-path out.json
|
||||
```
|
||||
|
||||
Code-first readers (`gorm`, `bun`, `drizzle`, `typeorm`) accept a single file or a
|
||||
directory of model files.
|
||||
|
||||
> Known issue: reading GORM models and writing DBML currently panics in the DBML
|
||||
> writer (`pkg/writers/dbml/writer.go`, `constraintToDBML`). Use another target
|
||||
> such as JSON until this is fixed.
|
||||
|
||||
## Cross-format combinations
|
||||
|
||||
```bash
|
||||
# ORM models to SQL DDL
|
||||
relspec convert --from gorm --from-path models.go --to pgsql --to-path schema.sql
|
||||
|
||||
# Prisma to Drizzle
|
||||
relspec convert --from prisma --from-path schema.prisma --to drizzle --to-path schema.ts
|
||||
|
||||
# DrawDB diagram to GraphQL
|
||||
relspec convert --from drawdb --from-path diagram.json --to graphql --to-path schema.graphql
|
||||
|
||||
# Merge several files while converting
|
||||
relspec convert --from json --from-list "a.json,b.json" --to yaml --to-path merged.yaml
|
||||
```
|
||||
|
||||
## Live databases
|
||||
|
||||
These need a reachable database:
|
||||
|
||||
```bash
|
||||
# PostgreSQL
|
||||
relspec convert --from pgsql --from-conn "postgres://user:pass@localhost:5432/mydb" \
|
||||
--to dbml --to-path schema.dbml
|
||||
|
||||
# SQL Server
|
||||
relspec convert --from mssql --from-conn "<mssql connection string>" \
|
||||
--to json --to-path schema.json
|
||||
|
||||
# SQLite database file (--from-conn takes the file path)
|
||||
relspec convert --from sqlite --from-conn ./app.db --to dbml --to-path schema.dbml
|
||||
```
|
||||
|
||||
## Formats outside `convert`
|
||||
|
||||
- `sqldir` (SQL script directory reader) and `sqlexec` (SQL execution writer) are
|
||||
used by `relspec scripts` and `relspec job`, and `sqldir` by `relspec diff`.
|
||||
See [SCRIPTS_COMMAND.md](SCRIPTS_COMMAND.md) and [JOB_FILES.md](JOB_FILES.md).
|
||||
- The `template` writer is exposed through `relspec templ`. See
|
||||
[TEMPLATE_MODE.md](TEMPLATE_MODE.md).
|
||||
+181
-28
@@ -9,10 +9,11 @@ relspec job run build-schema --plan # validate + print plan, execute nothing
|
||||
relspec job run build-schema # run the job (and its dependencies)
|
||||
```
|
||||
|
||||
## Design contract (first release)
|
||||
## Design contract
|
||||
|
||||
This is the smallest coherent contract that is safe and useful end to end.
|
||||
Anything not listed under "Supported" is intentionally deferred.
|
||||
This is a deliberately small, safe contract. Every capability is offline-testable
|
||||
except live database execution (`scripts-exec`), which is validated and planned
|
||||
offline and only connects at run time.
|
||||
|
||||
### Not a shell
|
||||
|
||||
@@ -24,13 +25,15 @@ means adding a vetted adapter in the RelSpec source.
|
||||
|----------------|--------------------------------------------------------------------|
|
||||
| `convert` | read one or more input schemas, additively merge them, write one output |
|
||||
| `merge` | like `convert` but requires ≥2 inputs and exposes `skip_*` merge options |
|
||||
| `split` | read one or more schemas, keep the selected schemas/tables, write one output |
|
||||
| `scripts-list` | deterministically list SQL scripts across one or more directories |
|
||||
| `scripts-exec` | execute SQL scripts across one or more directories against a live PostgreSQL database |
|
||||
| `templ` | apply a custom Go text template to one or more input schemas |
|
||||
| `inspect` | validate one or more schemas against rules and write a report |
|
||||
| `diff` | compare exactly two schemas and write a differences report |
|
||||
|
||||
Deferred (documented, not implemented here): `scripts` execution against a live
|
||||
database, `split`, `inspect`, `diff`, `templ`, job-to-job output wiring,
|
||||
log rotation/retention. Live SQL execution already exists as
|
||||
`relspec scripts execute`; wiring it into the job runner is a follow-up because
|
||||
it needs live database credentials and cannot be covered by offline tests.
|
||||
`convert`, `merge` and `split` are **producers**: their file output can be fed
|
||||
directly into another job with `from_job` (see below).
|
||||
|
||||
### Discovery and precedence
|
||||
|
||||
@@ -48,12 +51,20 @@ already forbid duplicate keys within a single file.
|
||||
|
||||
### Paths
|
||||
|
||||
* Every path (`inputs[].path`, `output.path`, `script_dirs[]`, `logfile`) is
|
||||
**relative to the directory containing the job file that declared the job**,
|
||||
not the process working directory.
|
||||
* Every path (`inputs[].path`, `output.path`, `report.path`, `rules`,
|
||||
`script_dirs[]`, `template`, `logfile`) is **relative to the directory
|
||||
containing the job file that declared the job**, not the process working
|
||||
directory.
|
||||
* Those path fields support `${NAME}` environment-variable references. They
|
||||
are expanded immediately before execution; a missing variable is an error.
|
||||
The expanded value is still subject to all relative-path and symlink safety
|
||||
checks.
|
||||
* Absolute paths, `~`-relative paths and any path that resolves outside the job
|
||||
file directory (`../`, `a/../../b`, …) are **rejected during validation** —
|
||||
before anything runs.
|
||||
* At run time each path is additionally resolved through its symlinks: a symlink
|
||||
inside the job-file directory that points outside it is rejected before the
|
||||
path is opened.
|
||||
|
||||
### Credentials
|
||||
|
||||
@@ -75,72 +86,140 @@ already forbid duplicate keys within a single file.
|
||||
first. Nothing is read, written, connected to, or executed if validation fails.
|
||||
Checks include:
|
||||
|
||||
* schema `version` (must be `1`), unknown YAML fields rejected
|
||||
* schema `version` — **forward-permissive**: any version `>= 1` is accepted.
|
||||
An omitted `version` is treated as the current one. A version newer than this
|
||||
build understands loads best-effort (unknown YAML fields are ignored and a
|
||||
warning is printed); at the current version unknown YAML fields are still
|
||||
rejected.
|
||||
* duplicate job names across files
|
||||
* unknown / missing `command`
|
||||
* per-command input/output shape (`convert`/`merge` need inputs + output;
|
||||
`scripts-list` needs `script_dirs` and forbids inputs/output)
|
||||
* per-command input/output shape:
|
||||
* `convert` needs ≥1 input + output; `merge` needs ≥2 inputs + output
|
||||
* `split` needs ≥1 input + a file output, plus an optional `select:` block
|
||||
* `scripts-list` needs `script_dirs` and forbids inputs/output
|
||||
* `scripts-exec` needs `script_dirs` and `output.conn_env` (pgsql only)
|
||||
* `inspect` needs ≥1 input + `report:` (format `markdown`|`json`)
|
||||
* `diff` needs **exactly 2** inputs + `report:` (format `summary`|`json`|`html`)
|
||||
* unknown input/output `format`
|
||||
* `from_job` targets exist, are producers (`convert`/`merge`/`split`) and write a
|
||||
single-file output
|
||||
* path traversal / absolute / home-relative paths
|
||||
* `depends_on` targets exist
|
||||
* dependency cycles (reported as `a -> b -> c -> a`)
|
||||
* `depends_on` and `from_job` targets exist
|
||||
* dependency cycles over the combined `depends_on` + `from_job` graph
|
||||
(reported as `a -> b -> c -> a`)
|
||||
|
||||
Then, immediately before running, per-job pre-flight resolves paths and checks:
|
||||
|
||||
* every input file exists and is a file
|
||||
* every input file exists and is a file (a `from_job` input is exempt — its
|
||||
producer runs earlier in the same plan)
|
||||
* every `script_dir` exists and is a directory
|
||||
* every `conn_env` variable is set
|
||||
* `output.path` does not already exist unless `output.overwrite: true`
|
||||
* `output.path` / `report.path` does not already exist unless the matching
|
||||
`overwrite: true` is set
|
||||
* `rules` (inspect), when given, exists and is a file
|
||||
* symlinks in every resolved path stay inside the job-file directory
|
||||
|
||||
If any pre-flight check fails for **any** job in the plan, **no** job runs.
|
||||
|
||||
### Execution and exit codes
|
||||
|
||||
* `relspec job run <name>` runs the job's `depends_on` closure first, in
|
||||
topological order (deterministic), then the job. `--no-deps` runs only the
|
||||
named job.
|
||||
* `relspec job run <name>` runs the job's dependency closure first
|
||||
(`depends_on` plus any `from_job` producers), in topological order
|
||||
(deterministic), then the job. `--no-deps` runs only the named job and is
|
||||
incompatible with `from_job` inputs.
|
||||
* `--dry-run` (alias `--plan`) prints the resolved plan and exits 0 without
|
||||
touching inputs, outputs or databases.
|
||||
* A failing job returns the underlying non-zero status (the process exits 1)
|
||||
and the error names the job. The logfile records `FAILED: <error>`; a
|
||||
successful job records `OK`. No separate success-marker file is written, so a
|
||||
failure can never leave a stale "success".
|
||||
* `inspect` fails the job when the report contains rule **errors** (enforced
|
||||
rules); warnings do not fail it. `diff` never fails on differences.
|
||||
* Single-file outputs and reports are written to a temporary file in the target
|
||||
directory and atomically renamed into place, so an interrupted run never
|
||||
leaves a partial file. Directory-emitting formats (`gorm`, `bun`, `drizzle`,
|
||||
`typeorm`, `prisma`) are written in place.
|
||||
|
||||
### Logfile rotation
|
||||
|
||||
When a job has a `logfile`, it is size-rotated before each run. Defaults are
|
||||
**5 MB** with **3** rotated files kept (`build.log` → `build.log.1` → …). Override
|
||||
per job with `log_max_size` / `log_keep`, or for a whole file with a top-level
|
||||
`defaults:` block. `log_max_size` accepts `B`/`KB`/`MB`/`GB` suffixes (e.g.
|
||||
`"512KB"`, `"5MB"`).
|
||||
|
||||
## Schema reference
|
||||
|
||||
```yaml
|
||||
version: 1 # required, must be 1
|
||||
version: 1 # optional; any value >= 1 is accepted
|
||||
defaults: # optional, file-wide
|
||||
log_max_size: 5MB # B / KB / MB / GB
|
||||
log_keep: 3
|
||||
jobs:
|
||||
<job-name>:
|
||||
command: convert | merge | scripts-list # required
|
||||
command: convert | merge | split | scripts-list | scripts-exec | templ | inspect | diff
|
||||
description: "free text" # optional, shown by `job list`
|
||||
depends_on: [other-job, ...] # optional
|
||||
inputs: # convert (≥1) / merge (≥2)
|
||||
inputs: # convert (≥1) / merge (≥2) / split (≥1) / inspect (≥1) / diff (exactly 2)
|
||||
- path: relative/file.dbml # file inputs
|
||||
format: dbml
|
||||
- format: pgsql # live-connection inputs
|
||||
conn_env: SOURCE_DB_URL # env var NAME
|
||||
script_dirs: # scripts-list (≥1)
|
||||
- from_job: build-schema # consume another job's file output
|
||||
script_dirs: # scripts-list / scripts-exec (≥1)
|
||||
- migrations/core
|
||||
- migrations/tenant
|
||||
output: # convert / merge (required)
|
||||
template: templates/schema.tmpl # templ (required)
|
||||
mode: table # templ: database/schema/script/table
|
||||
filename_pattern: "{{.Name}}.go" # templ multi-output modes
|
||||
select: # split (optional; default = keep everything)
|
||||
schemas: [public]
|
||||
tables: [users, orders]
|
||||
exclude_schemas: []
|
||||
exclude_tables: []
|
||||
database_name: SubsetDB # optional rename of the output database
|
||||
rules: .relspec-rules.yaml # inspect (optional; built-in defaults if omitted)
|
||||
report: # inspect (required) / diff (required)
|
||||
format: json # inspect: markdown|json ; diff: summary|json|html
|
||||
path: build/report.json # required, except a diff "summary" (goes to the log)
|
||||
overwrite: false
|
||||
output: # convert / merge / split (required); scripts-exec (required, conn_env)
|
||||
format: pgsql
|
||||
path: build/schema.sql # file output, OR:
|
||||
conn_env: TARGET_DB_URL # execute against DB (pgsql only)
|
||||
# default: reads live DB, executes only the diff
|
||||
# (file output = full DDL); set options.full_ddl to force full DDL
|
||||
overwrite: false # default false
|
||||
options:
|
||||
flatten_schema: false
|
||||
schema: public
|
||||
package: models # for gorm/bun output
|
||||
continue_on_error: false # pgsql output
|
||||
types: sqltypes # Bun/GORM nullable types: baselib|stdlib|sqltypes
|
||||
array_nullable: pointer_slice # Bun: slice|pointer_slice
|
||||
continue_on_error: false # pgsql / scripts-exec output
|
||||
full_ddl: false # pgsql database output: true = run full DDL; default diffs against live DB
|
||||
skip_relations: false # merge only
|
||||
skip_enums: false
|
||||
skip_views: false
|
||||
skip_domains: false
|
||||
skip_sequences: false
|
||||
logfile: .relspec/log/<job-name>.log # optional; appended to
|
||||
logfile: .relspec/log/<job-name>.log # optional; appended to, size-rotated
|
||||
log_max_size: 5MB # optional per-job override
|
||||
log_keep: 3 # optional per-job override
|
||||
```
|
||||
|
||||
For `templ`, `inputs` use the same file or `pgsql`/`conn_env` source forms as
|
||||
schema conversion. `output` is optional (empty means stdout); when present it
|
||||
contains `path` and `overwrite`, plus the explicit `format: text` marker. The
|
||||
marker is templ-only: it describes arbitrary text produced by a Go template,
|
||||
not a database schema format. It is optional for compatibility with older
|
||||
manifests and has no effect on template execution.
|
||||
|
||||
A `from_job` input takes no `path`, `format` or `conn_env`: it resolves to the
|
||||
named job's `output.path` and inherits its format, and implies a dependency on
|
||||
that job. The producer must be a `convert`, `merge` or `split` job writing a
|
||||
single-file output.
|
||||
|
||||
### Supported input formats
|
||||
|
||||
`dbml`, `dctx`, `drawdb`, `graphql`, `json`, `yaml`, `gorm`, `bun`, `drizzle`,
|
||||
@@ -221,3 +300,77 @@ jobs:
|
||||
path: snapshots/prod.dbml
|
||||
overwrite: true
|
||||
```
|
||||
|
||||
### Chain jobs with `from_job`, then lint the result
|
||||
|
||||
```yaml
|
||||
version: 1
|
||||
jobs:
|
||||
build-json:
|
||||
command: convert
|
||||
inputs:
|
||||
- { path: schema/core.dbml, format: dbml }
|
||||
- { path: schema/tenant.dbml, format: dbml }
|
||||
output: { format: json, path: build/schema.json, overwrite: true }
|
||||
lint-schema:
|
||||
command: inspect
|
||||
inputs:
|
||||
- from_job: build-json # implies depends_on: [build-json]
|
||||
rules: .relspec-rules.yaml # optional; built-in rules if omitted
|
||||
report:
|
||||
format: markdown
|
||||
path: build/lint-report.md
|
||||
overwrite: true
|
||||
```
|
||||
|
||||
`relspec job run lint-schema` runs `build-json` first, then inspects its output.
|
||||
The job fails (exit 1) if any enforced rule is violated.
|
||||
|
||||
### Split a subset out of a larger schema
|
||||
|
||||
```yaml
|
||||
version: 1
|
||||
jobs:
|
||||
posts-only:
|
||||
command: split
|
||||
inputs:
|
||||
- { path: schema/core.dbml, format: dbml }
|
||||
- { path: schema/tenant.dbml, format: dbml }
|
||||
select:
|
||||
tables: [posts]
|
||||
output: { format: dbml, path: build/posts.dbml, overwrite: true }
|
||||
```
|
||||
|
||||
### Diff two schemas
|
||||
|
||||
```yaml
|
||||
version: 1
|
||||
jobs:
|
||||
drift:
|
||||
command: diff
|
||||
inputs: # exactly two
|
||||
- { path: build/schema.json, format: json }
|
||||
- format: pgsql
|
||||
conn_env: PROD_DB_URL
|
||||
report:
|
||||
format: summary # summary → logfile; json/html need a path
|
||||
```
|
||||
|
||||
`diff` reports differences and always exits 0.
|
||||
|
||||
### Execute migration scripts against a live database
|
||||
|
||||
```yaml
|
||||
version: 1
|
||||
jobs:
|
||||
apply-migrations:
|
||||
command: scripts-exec
|
||||
script_dirs:
|
||||
- migrations/core
|
||||
- migrations/tenant
|
||||
output:
|
||||
conn_env: TARGET_DB_URL # pgsql only; no path
|
||||
options:
|
||||
continue_on_error: false
|
||||
logfile: .relspec/log/apply-migrations.log
|
||||
```
|
||||
|
||||
@@ -0,0 +1,212 @@
|
||||
# TUI mouse support plan
|
||||
|
||||
Issue: #46
|
||||
Status: design only; this document does not implement mouse input.
|
||||
|
||||
## 1. Current implementation and scope
|
||||
|
||||
The editor is created in `cmd/relspec/edit.go` by
|
||||
`ui.NewSchemaEditorWithConfigs(...).Run()`. `pkg/ui/editor.go` owns the
|
||||
`tview.Application`, `tview.Pages`, and application lifecycle. The current
|
||||
code never calls `Application.EnableMouse`, so tcell mouse reporting is off.
|
||||
The module uses tview v0.42.0 and tcell/v2 v2.13.9.
|
||||
|
||||
The first implementation should add `--no-mouse` to the `edit` Cobra command
|
||||
only. The flag is a local boolean, defaulting to false, and should be passed
|
||||
explicitly into the editor (prefer an options/config field rather than a
|
||||
package-global or environment variable). It must not affect convert, inspect,
|
||||
merge, or other commands. There is no environment-variable or persistent
|
||||
configuration setting in this issue: a command-line opt-out is predictable,
|
||||
visible in `edit --help`, and avoids adding configuration precedence rules.
|
||||
|
||||
At startup, the editor should call `app.EnableMouse(!noMouse)` before
|
||||
`Run()`. `--no-mouse` must mean that the application does not enable terminal
|
||||
mouse reporting and that no custom mouse handlers are relied upon. Keyboard
|
||||
behavior must remain identical in both modes.
|
||||
|
||||
Likely implementation files are `cmd/relspec/edit.go`,
|
||||
`pkg/ui/editor.go`, focused TUI mouse helpers/tests under `pkg/ui`, and a
|
||||
short user-facing note in the command help or TUI documentation. Do not
|
||||
refactor unrelated screens or data operations.
|
||||
|
||||
## 2. Widget and screen coverage
|
||||
|
||||
The application composes `Pages`, `Flex`, `TextView`, `List`, `Table`, `Form`,
|
||||
`Button`, `InputField`, `DropDown`, `TextArea`, `CheckBox`, and `Modal`.
|
||||
Vendored tview confirms mouse handlers exist for all of those relevant
|
||||
primitives, including focus on left-down, list/table selection, button clicks,
|
||||
form child dispatch, dropdown opening/drag selection, text-area cursor and
|
||||
scrolling, and modal button dispatch. `Pages`, `Flex`, and `Form` forward events
|
||||
to their children.
|
||||
|
||||
The coverage plan is:
|
||||
|
||||
* Main menu (`pkg/ui/main_menu.go`): left click focuses/selects a list entry;
|
||||
second activation opens it; buttons and exit confirmation remain reachable.
|
||||
* Schema, table, domain, object, relation, and database screens: click a row
|
||||
to select it; double-click the row to perform the same action as the
|
||||
keyboard Enter/selected callback where opening is meaningful; scroll lists
|
||||
and tables; click each action button.
|
||||
* Tables (`schema_screens.go`, `table_screens.go`, and object/relation tables):
|
||||
tview's table handler provides selection and scrolling, but it does not
|
||||
provide application-specific double-click activation. Add a small reusable
|
||||
wrapper/helper for the table instances that need it. It must preserve the
|
||||
existing selected row/column behavior and invoke the same callback as Enter,
|
||||
not duplicate mutation logic.
|
||||
* Forms (`load_save_screens.go` and the form-building screen files): click an
|
||||
input to focus it, click buttons to activate them, click a dropdown to open
|
||||
it and choose an option, scroll multiline help/text areas, and retain all
|
||||
existing keyboard Tab/Shift-Tab, shortcut, Enter, and Escape behavior.
|
||||
* Dialogs (`pkg/ui/dialogs.go` plus confirmation/error/success modals): modal
|
||||
buttons are clickable and the modal keeps focus above the underlying page.
|
||||
Clicking outside a modal must not activate the hidden page or dismiss a
|
||||
destructive confirmation. Escape and the existing button-key behavior stay
|
||||
authoritative.
|
||||
* The planned file browser and connection-string builder from issue #44 must
|
||||
use the same contracts: clickable entries/buttons and scrolling, with
|
||||
keyboard navigation and explicit cancel/accept paths. #46 should not
|
||||
implement #44's widgets; it should define the integration point and test
|
||||
them when #44 lands.
|
||||
|
||||
Do not promise drag semantics for every widget. Drag is appropriate for text
|
||||
selection/cursor movement and dropdown selection where tview already supports
|
||||
it. For ordinary list/table navigation, a click selects and the wheel scrolls;
|
||||
row dragging should not mutate data.
|
||||
|
||||
## 3. Exact mouse action contract
|
||||
|
||||
| Widget/type | Left down/click | Double click | Wheel/drag | Keyboard fallback |
|
||||
| --- | --- | --- | --- | --- |
|
||||
| Main/list menu | focus and select row | invoke row selected callback | scroll list | arrows, Enter, shortcuts |
|
||||
| Data table | focus and select cell/row | invoke the screen's existing open/edit action for the selected row | vertical/horizontal scroll as supported by tview | arrows, PageUp/PageDown, Enter, existing shortcuts |
|
||||
| Button | focus | same as one activation, never duplicate the callback | none | Tab/Shift-Tab, Enter/Space and existing shortcut |
|
||||
| Input field | focus; place cursor if supported | no destructive action | text-area behavior if provided by tview | typing, arrows, Home/End, Tab, Escape |
|
||||
| Text area/help | focus and position cursor | select word only where tview supports it; no application action | scroll; drag text selection if supported | arrows, PageUp/PageDown, standard editing keys |
|
||||
| Dropdown | focus/open and choose the hit option | same as click; no duplicate selection | drag through options only while open | arrows, Enter, Escape, Tab |
|
||||
| Checkbox | toggle on click | no second toggle | none | Space and existing form navigation |
|
||||
| Modal | focus/click visible button | same button action once | no underlying-page scrolling | Tab/Shift-Tab, Enter, Escape, existing button keys |
|
||||
| Blank/border/title area | focus containing primitive where useful | none | no mutation | current screen shortcuts |
|
||||
|
||||
Right and middle clicks should have no application action in the first
|
||||
release. Wheel events should be consumed only by the scrollable primitive
|
||||
under the pointer. Double-click timing/translation should come from tview/
|
||||
tcell; custom code must not fire the action once for both the click and the
|
||||
double-click. Any custom table wrapper needs a small state machine or tview
|
||||
mouse action handling that is tested for this property.
|
||||
|
||||
## 4. tview gaps and implementation boundaries
|
||||
|
||||
Enabling mouse support is not sufficient for the desired behavior. tview's
|
||||
built-in Table handler selects cells and scrolls but has no repository-specific
|
||||
row-open callback on double click. Existing screen code also wires keyboard
|
||||
input captures directly on individual widgets, so mouse actions must call the
|
||||
same screen callbacks rather than route through synthetic key events.
|
||||
|
||||
Use tview's `MouseHandler`/`WrapMouseHandler` contracts and `setFocus` rather
|
||||
than reading terminal coordinates in each screen. A reusable table adapter
|
||||
may embed `*tview.Table`, delegate ordinary actions to the original table
|
||||
handler, and add the screen's double-click callback. Keep the adapter in
|
||||
`pkg/ui` and use it only where a row-opening action exists. Do not modify the
|
||||
vendored tview copy.
|
||||
|
||||
The `Pages`/`Modal` dispatch order must be verified: a visible modal consumes
|
||||
its click before the page below it. Page transitions should happen only in the
|
||||
existing callbacks, so a stale hidden page cannot receive a click.
|
||||
|
||||
## 5. Keyboard, terminal, and copy/paste behavior
|
||||
|
||||
Mouse is an enhancement, never a requirement. Every acceptance path must be
|
||||
reachable with the existing keyboard controls, including load/save, navigation,
|
||||
editing, confirmations, cancel, and exit. `--no-mouse` is the regression mode
|
||||
for proving this contract.
|
||||
|
||||
Mouse reporting is terminal capability dependent. On local terminals it is
|
||||
negotiated by tcell; tmux and SSH can suppress, translate, or fail to pass
|
||||
mouse reporting depending on their configuration. The application must still
|
||||
start and remain keyboard usable if mouse reporting is unavailable or broken.
|
||||
Documentation should state that terminal/tmux configuration may be required,
|
||||
and that SSH behavior depends on the remote terminal path. Windows Terminal and
|
||||
other Windows console hosts should be treated as supported only insofar as the
|
||||
selected tcell backend reports mouse events; the CLI must not assume POSIX
|
||||
escape sequences or add platform-specific code in this issue.
|
||||
|
||||
Enabling mouse capture normally prevents terminal-native selection/copy from
|
||||
seeing ordinary button-drag events. Document the standard workaround: hold the
|
||||
terminal's bypass modifier (commonly Shift, terminal-dependent) for selection,
|
||||
or use `--no-mouse` when native copy/paste is the priority. Do not implement a
|
||||
second clipboard protocol. Input-field/text-area copy/paste must continue to
|
||||
use tview/tcell paste handling and keyboard shortcuts; verify that enabling
|
||||
mouse does not intercept paste events.
|
||||
|
||||
## 6. Test strategy using tcell simulation
|
||||
|
||||
Add focused tests rather than attempting a full interactive end-to-end test.
|
||||
Use `tcell.NewSimulationScreen("")`, `screen.Init()`, construct the editor or
|
||||
an isolated primitive tree, and inject events with the actual API:
|
||||
`SimulationScreen.InjectMouse(x, y, buttons, mod)` and `InjectKey(...)`.
|
||||
Coordinates must be derived from the primitive's drawn rectangle or fixed by a
|
||||
small deterministic test layout; do not use arbitrary coordinates without
|
||||
checking the rendered screen.
|
||||
|
||||
Minimum cases:
|
||||
|
||||
1. Default editor configuration enables mouse; the explicit disabled option
|
||||
leaves it disabled. If the Application API is not observable directly,
|
||||
test through the simulation screen's event path plus a constructor-level
|
||||
option assertion.
|
||||
2. A list click changes focus/selection, and double-click invokes the existing
|
||||
selected action exactly once.
|
||||
3. A table click selects the expected row/cell, wheel events change the visible
|
||||
offset, and double-click invokes the row action exactly once.
|
||||
4. Form button, input field, checkbox, and dropdown clicks match their
|
||||
keyboard callbacks.
|
||||
5. A modal button click acts on the modal and cannot activate the underlying
|
||||
page; Escape still cancels.
|
||||
6. `--no-mouse` leaves keyboard selection/activation unchanged and mouse
|
||||
injection has no application effect.
|
||||
7. Existing dialogs and screen transitions do not leave a stale mouse capture
|
||||
after a page is removed.
|
||||
|
||||
Prefer callback counters and selected-index assertions over screen-text-only
|
||||
assertions. Run the relevant `pkg/ui` tests with `go test -race ./pkg/ui` and
|
||||
run the full repository test suite if time/resources permit.
|
||||
|
||||
## 7. Rollout and acceptance criteria
|
||||
|
||||
Implementation is ready for review when:
|
||||
|
||||
* `relspec edit --help` documents `--no-mouse` and mouse is enabled by default.
|
||||
* Only the edit TUI is affected; non-TUI commands have no changed behavior.
|
||||
* Main screens, tables, lists, forms, dropdowns, buttons, text areas, and
|
||||
visible dialogs support the action contract above.
|
||||
* Keyboard-only operation is complete and verified with `--no-mouse`.
|
||||
* Modal clicks cannot fall through to an underlying page.
|
||||
* Table double-click behavior is explicit, tested, and does not duplicate
|
||||
activation.
|
||||
* tcell simulation tests cover default-on, opt-out, selection, scrolling,
|
||||
activation, dialog focus, and keyboard fallback.
|
||||
* `go test -race ./pkg/ui`, appropriate command tests, `go test ./...`,
|
||||
formatting, and `git diff --check` pass (or any limitation is recorded).
|
||||
* User-facing docs explain tmux/SSH/Windows variability and the terminal
|
||||
modifier workaround for native copy/paste.
|
||||
|
||||
Roll out in two implementation slices if needed: first application option,
|
||||
standard tview handlers, tests, and documentation; second only the reusable
|
||||
table double-click adapter and screen wiring. Do not block the first slice on
|
||||
issue #44, but do not claim #44's future widgets are covered until they use the
|
||||
same contract.
|
||||
|
||||
## 8. Open decisions and dependencies
|
||||
|
||||
* Confirm whether the project wants a public editor options type or a small
|
||||
`SetMouseEnabled`/constructor parameter; avoid a global flag.
|
||||
* Confirm the preferred terminal copy modifier in project documentation, since
|
||||
tmux, SSH clients, and Windows Terminal differ.
|
||||
* Decide whether horizontal wheel events should be supported where tview/table
|
||||
exposes them; vertical scrolling is mandatory, horizontal is optional.
|
||||
* Decide whether double-click opens every data table or only tables with an
|
||||
unambiguous row action. The plan recommends the latter.
|
||||
* Confirm #44's file-browser and connection-builder primitive choices before
|
||||
wiring their mouse tests.
|
||||
* Confirm CI has a stable non-terminal environment for simulation-screen tests;
|
||||
no real terminal, tmux session, database, or network should be required.
|
||||
@@ -0,0 +1,80 @@
|
||||
# RelSpec job examples
|
||||
|
||||
Run these commands from this directory:
|
||||
|
||||
```bash
|
||||
cd examples/jobs
|
||||
|
||||
# List every job from relspec.yml and the additional relspec.*.yml files.
|
||||
relspec job list
|
||||
|
||||
# Validate a job and print its resolved plan without reading or writing data.
|
||||
relspec job run build-schema --plan
|
||||
|
||||
# Build PostgreSQL DDL from the two DBML inputs.
|
||||
relspec job run build-schema
|
||||
|
||||
# Run a job's dependency chain. This runs build-schema first, then build-json.
|
||||
relspec job run build-json
|
||||
|
||||
# Inspect build-json's output. The from_job input adds the dependency automatically.
|
||||
relspec job run lint-schema
|
||||
|
||||
# Extract only the posts table.
|
||||
relspec job run posts-only
|
||||
|
||||
# List migration scripts in deterministic order without connecting to a database.
|
||||
relspec job run migration-order
|
||||
|
||||
# Render one TypeScript file per table using the example template.
|
||||
relspec job run generate-typescript
|
||||
|
||||
# Compare the two example schemas and write the summary to the logfile.
|
||||
relspec job run compare-schemas
|
||||
|
||||
# Use an environment variable in an output path.
|
||||
export VST=build/typescript
|
||||
relspec job run generate-typescript --plan
|
||||
relspec job run generate-typescript
|
||||
```
|
||||
|
||||
Database jobs use environment variables for connection strings. Set them in
|
||||
the shell before running the jobs; do not put connection strings in YAML:
|
||||
|
||||
```bash
|
||||
export SOURCE_DB_URL='postgres://user:password@localhost/source_db'
|
||||
export TARGET_DB_URL='postgres://user:password@localhost/target_db'
|
||||
|
||||
# Read the source database and write a local DBML snapshot.
|
||||
relspec job run snapshot-database
|
||||
|
||||
# Preview migration execution without connecting to PostgreSQL.
|
||||
relspec job run apply-migrations --plan
|
||||
|
||||
# Execute the migrations against TARGET_DB_URL.
|
||||
relspec job run apply-migrations
|
||||
```
|
||||
|
||||
The `apply-migrations` job demonstrates execution from multiple directories:
|
||||
|
||||
```yaml
|
||||
script_dirs:
|
||||
- migrations/core
|
||||
- migrations/tenant
|
||||
output:
|
||||
format: pgsql
|
||||
conn_env: TARGET_DB_URL
|
||||
```
|
||||
|
||||
RelSpec combines the SQL files from both folders in deterministic order before
|
||||
executing them against the configured PostgreSQL database.
|
||||
|
||||
To use a different directory or manifest, pass `--dir` or `--file`:
|
||||
|
||||
```bash
|
||||
relspec job list --dir /path/to/project
|
||||
relspec job run build-schema --file /path/to/project/relspec.yml --plan
|
||||
```
|
||||
|
||||
See [../../docs/JOB_FILES.md](../../docs/JOB_FILES.md) for the complete job
|
||||
file reference.
|
||||
@@ -0,0 +1,29 @@
|
||||
# Database-oriented examples. Set the referenced environment variables before
|
||||
# running these jobs; connection strings never belong in a job file.
|
||||
version: 1
|
||||
|
||||
jobs:
|
||||
snapshot-database:
|
||||
command: convert
|
||||
description: Read a PostgreSQL database and write a DBML snapshot
|
||||
inputs:
|
||||
- format: pgsql
|
||||
conn_env: SOURCE_DB_URL
|
||||
output:
|
||||
format: dbml
|
||||
path: build/database-snapshot.dbml
|
||||
overwrite: true
|
||||
|
||||
apply-migrations:
|
||||
command: scripts-exec
|
||||
description: Execute migrations from core and tenant folders against PostgreSQL
|
||||
# All SQL files from these directories are discovered in deterministic order.
|
||||
script_dirs:
|
||||
- migrations/core
|
||||
- migrations/tenant
|
||||
output:
|
||||
format: pgsql
|
||||
conn_env: TARGET_DB_URL
|
||||
options:
|
||||
continue_on_error: false
|
||||
logfile: .relspec/log/apply-migrations.log
|
||||
@@ -0,0 +1,32 @@
|
||||
# Reporting and templating examples. These jobs are discovered together with
|
||||
# relspec.yml when running `relspec job list` from this directory.
|
||||
version: 1
|
||||
|
||||
jobs:
|
||||
generate-typescript:
|
||||
command: templ
|
||||
description: Render one TypeScript file per table from the merged schema
|
||||
inputs:
|
||||
- path: schema/core.dbml
|
||||
format: dbml
|
||||
- path: schema/tenant.dbml
|
||||
format: dbml
|
||||
template: templates/schema.tmpl
|
||||
mode: table
|
||||
filename_pattern: "{{.Name}}.ts"
|
||||
output:
|
||||
format: text
|
||||
path: "${VST}"
|
||||
overwrite: true
|
||||
|
||||
compare-schemas:
|
||||
command: diff
|
||||
description: Compare the two example schemas and print a summary
|
||||
inputs:
|
||||
- path: schema/core.dbml
|
||||
format: dbml
|
||||
- path: schema/tenant.dbml
|
||||
format: dbml
|
||||
report:
|
||||
format: summary
|
||||
logfile: .relspec/log/compare-schemas.log
|
||||
@@ -4,7 +4,14 @@
|
||||
# relspec job list
|
||||
# relspec job run build-schema --plan
|
||||
# relspec job run build-schema
|
||||
# relspec job run lint-schema # inspect, consuming build-json's output
|
||||
version: 1
|
||||
|
||||
# File-wide defaults. Individual jobs may override log_max_size / log_keep.
|
||||
defaults:
|
||||
log_max_size: 2MB
|
||||
log_keep: 5
|
||||
|
||||
jobs:
|
||||
build-schema:
|
||||
command: convert
|
||||
@@ -43,3 +50,30 @@ jobs:
|
||||
- migrations/core
|
||||
- migrations/tenant
|
||||
logfile: .relspec/log/migration-order.log
|
||||
|
||||
lint-schema:
|
||||
command: inspect
|
||||
description: Validate build-json's output against the built-in rules
|
||||
# No depends_on needed: the from_job input implies a dependency on build-json.
|
||||
inputs:
|
||||
- from_job: build-json
|
||||
report:
|
||||
format: markdown
|
||||
path: build/lint-report.md
|
||||
overwrite: true
|
||||
logfile: .relspec/log/lint-schema.log
|
||||
|
||||
posts-only:
|
||||
command: split
|
||||
description: Extract just the posts table into its own DBML file
|
||||
inputs:
|
||||
- path: schema/core.dbml
|
||||
format: dbml
|
||||
- path: schema/tenant.dbml
|
||||
format: dbml
|
||||
select:
|
||||
tables: [posts]
|
||||
output:
|
||||
format: dbml
|
||||
path: build/posts.dbml
|
||||
overwrite: true
|
||||
|
||||
@@ -0,0 +1,6 @@
|
||||
// Code generated by RelSpec. DO NOT EDIT.
|
||||
export interface {{.Name}} {
|
||||
{{- range values .Table.Columns}}
|
||||
{{.Name}}: {{.Type}};
|
||||
{{- end}}
|
||||
}
|
||||
@@ -1,9 +1,10 @@
|
||||
module git.warky.dev/wdevs/relspecgo
|
||||
|
||||
go 1.25.7
|
||||
go 1.25.13
|
||||
|
||||
require (
|
||||
github.com/gdamore/tcell/v2 v2.13.9
|
||||
github.com/go-sql-driver/mysql v1.9.3
|
||||
github.com/google/uuid v1.6.0
|
||||
github.com/jackc/pgx/v5 v5.9.2
|
||||
github.com/microsoft/go-mssqldb v1.10.0
|
||||
@@ -12,12 +13,13 @@ require (
|
||||
github.com/stretchr/testify v1.11.1
|
||||
github.com/uptrace/bun v1.2.18
|
||||
github.com/uptrace/bun/dialect/pgdialect v1.2.18
|
||||
golang.org/x/text v0.37.0
|
||||
golang.org/x/text v0.39.0
|
||||
gopkg.in/yaml.v3 v3.0.1
|
||||
modernc.org/sqlite v1.50.1
|
||||
)
|
||||
|
||||
require (
|
||||
filippo.io/edwards25519 v1.1.0 // indirect
|
||||
github.com/davecgh/go-spew v1.1.1 // indirect
|
||||
github.com/dustin/go-humanize v1.0.1 // indirect
|
||||
github.com/gdamore/encoding v1.0.1 // indirect
|
||||
@@ -42,11 +44,11 @@ require (
|
||||
github.com/tmthrgd/go-hex v0.0.0-20190904060850-447a3041c3bc // indirect
|
||||
github.com/vmihailenco/msgpack/v5 v5.4.1 // indirect
|
||||
github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect
|
||||
golang.org/x/crypto v0.51.0 // indirect
|
||||
golang.org/x/sync v0.20.0 // indirect
|
||||
golang.org/x/sys v0.44.0 // indirect
|
||||
golang.org/x/term v0.43.0 // indirect
|
||||
golang.org/x/tools v0.45.0 // indirect
|
||||
golang.org/x/crypto v0.53.0 // indirect
|
||||
golang.org/x/net v0.56.0 // indirect
|
||||
golang.org/x/sync v0.21.0 // indirect
|
||||
golang.org/x/sys v0.46.0 // indirect
|
||||
golang.org/x/term v0.44.0 // indirect
|
||||
modernc.org/libc v1.72.3 // indirect
|
||||
modernc.org/mathutil v1.7.1 // indirect
|
||||
modernc.org/memory v1.11.0 // indirect
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA=
|
||||
filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.21.1 h1:jHb/wfvRikGdxMXYV3QG/SzUOPYN9KEUUuC0Yd0/vC0=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.21.1/go.mod h1:pzBXCYn05zvYIrwLgtK8Ap8QcjRg+0i76tMQdWN6wOk=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.13.1 h1:Hk5QBxZQC1jb2Fwj6mpzme37xbCDdNTxU7O9eb5+LB4=
|
||||
@@ -21,6 +23,8 @@ github.com/gdamore/encoding v1.0.1 h1:YzKZckdBL6jVt2Gc+5p82qhrGiqMdG/eNs6Wy0u3Uh
|
||||
github.com/gdamore/encoding v1.0.1/go.mod h1:0Z0cMFinngz9kS1QfMjCP8TY7em3bZYeeklsSDPivEo=
|
||||
github.com/gdamore/tcell/v2 v2.13.9 h1:uI5l3DYPcFvHINKlGft+en23evOKL+dwtD21QR8ejVA=
|
||||
github.com/gdamore/tcell/v2 v2.13.9/go.mod h1:+Wfe208WDdB7INEtCsNrAN6O2m+wsTPk1RAovjaILlo=
|
||||
github.com/go-sql-driver/mysql v1.9.3 h1:U/N249h2WzJ3Ukj8SowVFjdtZKfu9vlLZxjPXV1aweo=
|
||||
github.com/go-sql-driver/mysql v1.9.3/go.mod h1:qn46aNg1333BRMNU69Lq93t8du/dwxI64Gl8i5p1WMU=
|
||||
github.com/golang-jwt/jwt/v5 v5.3.1 h1:kYf81DTWFe7t+1VvL7eS+jKFVWaUnK9cB1qbwn63YCY=
|
||||
github.com/golang-jwt/jwt/v5 v5.3.1/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE=
|
||||
github.com/golang-sql/civil v0.0.0-20220223132316-b832511892a9 h1:au07oEsX2xN0ktxqI+Sida1w446QrXBRJ0nee3SNZlA=
|
||||
@@ -102,49 +106,49 @@ github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5t
|
||||
go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=
|
||||
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
||||
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
|
||||
golang.org/x/crypto v0.51.0 h1:IBPXwPfKxY7cWQZ38ZCIRPI50YLeevDLlLnyC5wRGTI=
|
||||
golang.org/x/crypto v0.51.0/go.mod h1:8AdwkbraGNABw2kOX6YFPs3WM22XqI4EXEd8g+x7Oc8=
|
||||
golang.org/x/crypto v0.53.0 h1:QZ4Muo8THX6CizN2vPPd5fBGHyogrdK9fG4wLPFUsto=
|
||||
golang.org/x/crypto v0.53.0/go.mod h1:DNLU434OwVakk9PzuwV8w62mAJpRJL3vsgcfp4Qnsio=
|
||||
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
|
||||
golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
|
||||
golang.org/x/mod v0.36.0 h1:JJjpVx6myfUsUdAzZuOSTTmRE0PfZeNWzzvKrP7amb4=
|
||||
golang.org/x/mod v0.36.0/go.mod h1:moc6ELqsWcOw5Ef3xVprK5ul/MvtVvkIXLziUOICjUQ=
|
||||
golang.org/x/mod v0.37.0 h1:vF1DjpVEshcIqoEaauuHebaLk1O1forxjxBaVn884JQ=
|
||||
golang.org/x/mod v0.37.0/go.mod h1:m8S8VeM9r4dzDwjrKO0a1sZP3YjeMamRRlD+fmR2Q/0=
|
||||
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
|
||||
golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
|
||||
golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
|
||||
golang.org/x/net v0.54.0 h1:2zJIZAxAHV/OHCDTCOHAYehQzLfSXuf/5SoL/Dv6w/w=
|
||||
golang.org/x/net v0.54.0/go.mod h1:Sj4oj8jK6XmHpBZU/zWHw3BV3abl4Kvi+Ut7cQcY+cQ=
|
||||
golang.org/x/net v0.56.0 h1:Rw8j/hFzGvJUZwNBXnAtf5sVDVt+65SK2C7IxCxZt5o=
|
||||
golang.org/x/net v0.56.0/go.mod h1:D3Ku6r+V6JROoZK144D2XfMHFcMq/0zSfLelVTCFKec=
|
||||
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4=
|
||||
golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
|
||||
golang.org/x/sync v0.21.0 h1:HLII4xRRTtCRkxYp4HNFF0Js/Og6q2i++KXbg0gHCwM=
|
||||
golang.org/x/sync v0.21.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
|
||||
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.44.0 h1:ildZl3J4uzeKP07r2F++Op7E9B29JRUy+a27EibtBTQ=
|
||||
golang.org/x/sys v0.44.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
|
||||
golang.org/x/sys v0.46.0 h1:noSf2Fq6F8DBgS+LysIkx7rIExoNHJsxOAtPp4rthXw=
|
||||
golang.org/x/sys v0.46.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
|
||||
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
|
||||
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
|
||||
golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
|
||||
golang.org/x/term v0.43.0 h1:S4RLU2sB31O/NCl+zFN9Aru9A/Cq2aqKpTZJ6B+DwT4=
|
||||
golang.org/x/term v0.43.0/go.mod h1:lrhlHNdQJHO+1qVYiHfFKVuVioJIheAc3fBSMFYEIsk=
|
||||
golang.org/x/term v0.44.0 h1:0rLvDRCtNj0gZkyIXhCyOb2OAzEhLVqc4B+hrsBhrmc=
|
||||
golang.org/x/term v0.44.0/go.mod h1:7ze4MdzUzLXpSAoFP1H0bOI9aXDqveSvatT5vKcFh2Y=
|
||||
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
||||
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
|
||||
golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
|
||||
golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
|
||||
golang.org/x/text v0.37.0 h1:Cqjiwd9eSg8e0QAkyCaQTNHFIIzWtidPahFWR83rTrc=
|
||||
golang.org/x/text v0.37.0/go.mod h1:a5sjxXGs9hsn/AJVwuElvCAo9v8QYLzvavO5z2PiM38=
|
||||
golang.org/x/text v0.39.0 h1:UbZz4pLOvn600D6Oh6GGEI6VAmndrEBLv8/6BEXzyus=
|
||||
golang.org/x/text v0.39.0/go.mod h1:3UwRclnC2g0TU9x8PZiyfOajCd1zaUNHF9cvqcQZ+ZM=
|
||||
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
||||
golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
|
||||
golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
|
||||
golang.org/x/tools v0.45.0 h1:18qN3FAooORvApf5XjCXgsuayZOEtXf6JK18I3+ONa8=
|
||||
golang.org/x/tools v0.45.0/go.mod h1:LuUGqqaXcXMEFEruIVJVm5mgDD8vww/z/SR1gQ4uE/0=
|
||||
golang.org/x/tools v0.47.0 h1:7Kn5x/d1svx/PzryTsqeoZN4TZwqeH5pGWjefhLi/1Q=
|
||||
golang.org/x/tools v0.47.0/go.mod h1:dFHnyTvFWY212G+h7ZY4Vsp/K3U4/7W9TyVaAul8uCA=
|
||||
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
|
||||
|
||||
+2
-2
@@ -1,6 +1,6 @@
|
||||
# Maintainer: Hein (Warky Devs) <hein@warky.dev>
|
||||
pkgname=relspec
|
||||
pkgver=1.0.74
|
||||
pkgver=1.0.85
|
||||
pkgrel=1
|
||||
pkgdesc="RelSpec is a comprehensive database relations management tool that reads, transforms, and writes database table specifications across multiple formats and ORMs."
|
||||
arch=('x86_64' 'aarch64')
|
||||
@@ -15,7 +15,7 @@ build() {
|
||||
export CGO_ENABLED=0
|
||||
go build \
|
||||
-trimpath \
|
||||
-ldflags "-X git.warky.dev/wdevs/relspecgo/cmd/relspec.version=$pkgver" \
|
||||
-ldflags "-X 'git.warky.dev/wdevs/relspecgo/pkg/buildinfo.Version=v$pkgver' -X 'git.warky.dev/wdevs/relspecgo/pkg/buildinfo.BuildDate=$(date -u +"%Y-%m-%d %H:%M:%S UTC")'" \
|
||||
-o "$pkgname" ./cmd/relspec
|
||||
}
|
||||
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
Name: relspec
|
||||
Version: 1.0.74
|
||||
Version: 1.0.85
|
||||
Release: 1%{?dist}
|
||||
Summary: RelSpec is a comprehensive database relations management tool that reads, transforms, and writes database table specifications across multiple formats and ORMs.
|
||||
|
||||
@@ -25,7 +25,7 @@ DBML, GraphQL, and more.
|
||||
export CGO_ENABLED=0
|
||||
go build \
|
||||
-trimpath \
|
||||
-ldflags "-X git.warky.dev/wdevs/relspecgo/cmd/relspec.version=%{version}" \
|
||||
-ldflags "-X 'git.warky.dev/wdevs/relspecgo/pkg/buildinfo.Version=v%{version}' -X 'git.warky.dev/wdevs/relspecgo/pkg/buildinfo.BuildDate=$(date -u +"%%Y-%%m-%%d %%H:%%M:%%S UTC")'" \
|
||||
-o %{name} ./cmd/relspec
|
||||
|
||||
%install
|
||||
|
||||
@@ -137,7 +137,10 @@ func TestScanDir_OrdersByPriorityThenSequence(t *testing.T) {
|
||||
t.Fatalf("expected 4 items, got %d", len(items))
|
||||
}
|
||||
|
||||
type ps struct{ p int; s uint }
|
||||
type ps struct {
|
||||
p int
|
||||
s uint
|
||||
}
|
||||
want := []ps{{1, 1}, {1, 2}, {2, 1}, {2, 2}}
|
||||
for i, w := range want {
|
||||
got := ps{items[i].Priority, items[i].Sequence}
|
||||
|
||||
@@ -0,0 +1,65 @@
|
||||
// Package buildinfo exposes the RelSpec version and build date so that both the
|
||||
// CLI and the schema writers can stamp generated output with the same values.
|
||||
package buildinfo
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"runtime/debug"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Version and BuildDate are set via -ldflags at build time (see Makefile). When
|
||||
// built without ldflags they are backfilled from the Go module build info.
|
||||
var (
|
||||
Version = "dev"
|
||||
BuildDate = "unknown"
|
||||
)
|
||||
|
||||
func init() {
|
||||
if Version != "dev" {
|
||||
return
|
||||
}
|
||||
info, ok := debug.ReadBuildInfo()
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
var rev, vcsTime string
|
||||
for _, s := range info.Settings {
|
||||
switch s.Key {
|
||||
case "vcs.revision":
|
||||
if len(s.Value) >= 7 {
|
||||
rev = s.Value[:7]
|
||||
}
|
||||
case "vcs.time":
|
||||
vcsTime = s.Value
|
||||
}
|
||||
}
|
||||
if rev != "" {
|
||||
Version = rev
|
||||
}
|
||||
if t, err := time.Parse(time.RFC3339, vcsTime); err == nil {
|
||||
BuildDate = t.UTC().Format("2006-01-02 15:04:05 UTC")
|
||||
}
|
||||
}
|
||||
|
||||
// GeneratedComment returns the one-line provenance string embedded in generated
|
||||
// files, e.g. "RelSpec dev (built: unknown)".
|
||||
func GeneratedComment() string {
|
||||
return fmt.Sprintf("RelSpec %s (built: %s)", Version, BuildDate)
|
||||
}
|
||||
|
||||
const AsciiLogo = `
|
||||
██████╗ ███████╗██╗ ███████╗██████╗ ███████╗ ██████╗
|
||||
██╔══██╗██╔════╝██║ ██╔════╝██╔══██╗██╔════╝██╔════╝
|
||||
██████╔╝█████╗ ██║ ███████╗██████╔╝█████╗ ██║
|
||||
██╔══██╗██╔══╝ ██║ ╚════██║██╔═══╝ ██╔══╝ ██║
|
||||
██║ ██║███████╗███████╗███████║██║ ███████╗╚██████╗
|
||||
╚═╝ ╚═╝╚══════╝╚══════╝╚══════╝╚═╝ ╚══════╝ ╚═════╝
|
||||
[ IN ] ──▶ [ RELSPEC ] ──▶ [ OUT ]
|
||||
╔══════════════════════════════════════╗
|
||||
║ ║
|
||||
║ © WARKY DEVS ║
|
||||
║ Author: Hein (hein@warky.dev) ║
|
||||
║ ║
|
||||
╚══════════════════════════════════════╝
|
||||
`
|
||||
+3
-5
@@ -264,7 +264,7 @@ func compareColumnDetails(source, target *models.Column) map[string]any {
|
||||
|
||||
// comparableColumn accepts DBML's compact type/default spelling as well as
|
||||
// PostgreSQL's normalized fields (for example varchar(255) vs varchar + 255).
|
||||
func comparableColumn(column *models.Column) (string, int, any) {
|
||||
func comparableColumn(column *models.Column) (normalizedType string, length int, defaultVal any) {
|
||||
typeName := strings.TrimSpace(column.Type)
|
||||
defaultValue := column.Default
|
||||
lower := strings.ToLower(typeName)
|
||||
@@ -274,7 +274,7 @@ func comparableColumn(column *models.Column) (string, int, any) {
|
||||
}
|
||||
typeName = strings.TrimSpace(typeName[:i])
|
||||
}
|
||||
length := column.Length
|
||||
length = column.Length
|
||||
if open := strings.LastIndex(typeName, "("); open >= 0 && strings.HasSuffix(typeName, ")") {
|
||||
if parsed, err := strconv.Atoi(strings.TrimSpace(typeName[open+1 : len(typeName)-1])); err == nil && length == 0 {
|
||||
length = parsed
|
||||
@@ -353,9 +353,7 @@ func compareIndexes(source, target map[string]*models.Index) *IndexDiff {
|
||||
}
|
||||
|
||||
for _, key := range sortedKeys(remainingTarget) {
|
||||
for _, index := range remainingTarget[key] {
|
||||
diff.Extra = append(diff.Extra, index)
|
||||
}
|
||||
diff.Extra = append(diff.Extra, remainingTarget[key]...)
|
||||
}
|
||||
return diff
|
||||
}
|
||||
|
||||
@@ -130,7 +130,6 @@ func TestFormatSummary(t *testing.T) {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
var buf bytes.Buffer
|
||||
err := formatSummary(tt.result, &buf)
|
||||
|
||||
if err != nil {
|
||||
t.Errorf("formatSummary() error = %v", err)
|
||||
return
|
||||
@@ -159,7 +158,6 @@ func TestFormatJSON(t *testing.T) {
|
||||
|
||||
var buf bytes.Buffer
|
||||
err := formatJSON(result, &buf)
|
||||
|
||||
if err != nil {
|
||||
t.Errorf("formatJSON() error = %v", err)
|
||||
return
|
||||
@@ -288,7 +286,6 @@ func TestFormatHTML(t *testing.T) {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
var buf bytes.Buffer
|
||||
err := formatHTML(tt.result, &buf)
|
||||
|
||||
if err != nil {
|
||||
t.Errorf("formatHTML() error = %v", err)
|
||||
return
|
||||
@@ -334,7 +331,6 @@ func TestFormatSummaryWithColumns(t *testing.T) {
|
||||
|
||||
var buf bytes.Buffer
|
||||
err := formatSummary(result, &buf)
|
||||
|
||||
if err != nil {
|
||||
t.Errorf("formatSummary() error = %v", err)
|
||||
return
|
||||
@@ -383,7 +379,6 @@ func TestFormatSummaryWithIndexes(t *testing.T) {
|
||||
|
||||
var buf bytes.Buffer
|
||||
err := formatSummary(result, &buf)
|
||||
|
||||
if err != nil {
|
||||
t.Errorf("formatSummary() error = %v", err)
|
||||
return
|
||||
@@ -425,7 +420,6 @@ func TestFormatSummaryWithConstraints(t *testing.T) {
|
||||
|
||||
var buf bytes.Buffer
|
||||
err := formatSummary(result, &buf)
|
||||
|
||||
if err != nil {
|
||||
t.Errorf("formatSummary() error = %v", err)
|
||||
return
|
||||
@@ -448,7 +442,6 @@ func TestFormatJSONIndentation(t *testing.T) {
|
||||
|
||||
var buf bytes.Buffer
|
||||
err := formatJSON(result, &buf)
|
||||
|
||||
if err != nil {
|
||||
t.Errorf("formatJSON() error = %v", err)
|
||||
return
|
||||
|
||||
@@ -123,6 +123,7 @@ rules:
|
||||
| `missing_primary_key` | `have_primary_key` | Ensure tables have primary keys |
|
||||
| `orphaned_foreign_key` | `orphaned_foreign_key` | Detect FKs referencing non-existent tables |
|
||||
| `circular_dependency` | `circular_dependency` | Detect circular FK dependencies |
|
||||
| `duplicate_index_name` | `duplicate_index_name` | Index / PK / unique names must be unique per schema (default: `enforce`) |
|
||||
|
||||
## Rule Configuration
|
||||
|
||||
|
||||
@@ -161,6 +161,7 @@ func getValidator(functionName string) (validatorFunc, bool) {
|
||||
"have_primary_key": validateMissingPrimaryKey,
|
||||
"orphaned_foreign_key": validateOrphanedForeignKey,
|
||||
"circular_dependency": validateCircularDependency,
|
||||
"duplicate_index_name": validateDuplicateIndexName,
|
||||
}
|
||||
|
||||
fn, exists := validators[functionName]
|
||||
@@ -168,7 +169,7 @@ func getValidator(functionName string) (validatorFunc, bool) {
|
||||
}
|
||||
|
||||
// createResult is a helper to create a validation result
|
||||
func createResult(ruleName string, passed bool, message string, location string, context map[string]interface{}) ValidationResult {
|
||||
func createResult(ruleName string, passed bool, message, location string, context map[string]interface{}) ValidationResult {
|
||||
return ValidationResult{
|
||||
RuleName: ruleName,
|
||||
Message: message,
|
||||
|
||||
@@ -29,7 +29,6 @@ func TestInspect(t *testing.T) {
|
||||
|
||||
inspector := NewInspector(db, config)
|
||||
report, err := inspector.Inspect()
|
||||
|
||||
if err != nil {
|
||||
t.Fatalf("Inspect() returned error: %v", err)
|
||||
}
|
||||
@@ -103,7 +102,6 @@ func TestInspectWithDisabledRules(t *testing.T) {
|
||||
|
||||
inspector := NewInspector(db, config)
|
||||
report, err := inspector.Inspect()
|
||||
|
||||
if err != nil {
|
||||
t.Fatalf("Inspect() with disabled rules returned error: %v", err)
|
||||
}
|
||||
@@ -135,7 +133,6 @@ func TestInspectWithEnforcedRules(t *testing.T) {
|
||||
|
||||
inspector := NewInspector(db, config)
|
||||
report, err := inspector.Inspect()
|
||||
|
||||
if err != nil {
|
||||
t.Fatalf("Inspect() returned error: %v", err)
|
||||
}
|
||||
|
||||
@@ -141,7 +141,7 @@ func (f *MarkdownFormatter) formatHeader(text string) string {
|
||||
return f.formatBold("# " + text)
|
||||
}
|
||||
|
||||
func (f *MarkdownFormatter) formatSubheader(text string, color string) string {
|
||||
func (f *MarkdownFormatter) formatSubheader(text, color string) string {
|
||||
header := "### " + text
|
||||
if f.UseColors {
|
||||
return color + colorBold + header + colorReset
|
||||
@@ -156,7 +156,7 @@ func (f *MarkdownFormatter) formatBold(text string) string {
|
||||
return "**" + text + "**"
|
||||
}
|
||||
|
||||
func (f *MarkdownFormatter) colorize(text string, color string) string {
|
||||
func (f *MarkdownFormatter) colorize(text, color string) string {
|
||||
if f.UseColors {
|
||||
return color + text + colorReset
|
||||
}
|
||||
|
||||
@@ -154,6 +154,11 @@ func GetDefaultConfig() *Config {
|
||||
Function: "circular_dependency",
|
||||
Message: "Circular foreign key dependency detected",
|
||||
},
|
||||
"duplicate_index_name": {
|
||||
Enabled: "enforce",
|
||||
Function: "duplicate_index_name",
|
||||
Message: "Index name is reused within the schema; PostgreSQL skips the duplicate CREATE INDEX IF NOT EXISTS",
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
@@ -37,6 +37,7 @@ func TestGetDefaultConfig(t *testing.T) {
|
||||
"missing_primary_key",
|
||||
"orphaned_foreign_key",
|
||||
"circular_dependency",
|
||||
"duplicate_index_name",
|
||||
}
|
||||
|
||||
for _, ruleName := range expectedRules {
|
||||
@@ -49,7 +50,6 @@ func TestGetDefaultConfig(t *testing.T) {
|
||||
func TestLoadConfig_NonExistentFile(t *testing.T) {
|
||||
// Try to load a non-existent file
|
||||
config, err := LoadConfig("/path/to/nonexistent/file.yaml")
|
||||
|
||||
if err != nil {
|
||||
t.Fatalf("LoadConfig() with non-existent file returned error: %v", err)
|
||||
}
|
||||
@@ -83,7 +83,7 @@ rules:
|
||||
message: "Table name too long"
|
||||
`
|
||||
|
||||
err := os.WriteFile(configPath, []byte(configContent), 0644)
|
||||
err := os.WriteFile(configPath, []byte(configContent), 0o644)
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to create test config file: %v", err)
|
||||
}
|
||||
@@ -133,7 +133,7 @@ func TestLoadConfig_InvalidYAML(t *testing.T) {
|
||||
|
||||
invalidContent := `invalid: yaml: content: {[}]`
|
||||
|
||||
err := os.WriteFile(configPath, []byte(invalidContent), 0644)
|
||||
err := os.WriteFile(configPath, []byte(invalidContent), 0o644)
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to create test config file: %v", err)
|
||||
}
|
||||
|
||||
@@ -643,3 +643,64 @@ func contains(slice []string, value string) bool {
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// validateDuplicateIndexName checks that index names are unique per schema.
|
||||
// PostgreSQL keeps indexes in the schema-wide relation namespace, so a name
|
||||
// reused on another table makes CREATE INDEX IF NOT EXISTS silently skip it.
|
||||
// Primary key and unique constraints create backing indexes and share that
|
||||
// namespace too. An index and a constraint with the same name on the same
|
||||
// table describe one object and are not reported.
|
||||
func validateDuplicateIndexName(db *models.Database, rule Rule, ruleName string) []ValidationResult {
|
||||
results := []ValidationResult{}
|
||||
|
||||
for _, schema := range db.Schemas {
|
||||
// lowercased name -> "table" entries, one per distinct object
|
||||
owners := make(map[string][]string)
|
||||
display := make(map[string]string)
|
||||
order := []string{}
|
||||
|
||||
add := func(name, table string, sameTableMerges bool) {
|
||||
key := strings.ToLower(name)
|
||||
if _, seen := owners[key]; !seen {
|
||||
order = append(order, key)
|
||||
display[key] = name
|
||||
}
|
||||
if sameTableMerges && contains(owners[key], table) {
|
||||
return
|
||||
}
|
||||
owners[key] = append(owners[key], table)
|
||||
}
|
||||
|
||||
for _, table := range schema.Tables {
|
||||
for _, key := range sortedKeys(table.Indexes) {
|
||||
if name := table.Indexes[key].Name; name != "" {
|
||||
add(name, table.Name, false)
|
||||
}
|
||||
}
|
||||
for _, c := range sortConstraints(table.Constraints) {
|
||||
if c.Name == "" || (c.Type != models.PrimaryKeyConstraint && c.Type != models.UniqueConstraint) {
|
||||
continue
|
||||
}
|
||||
add(c.Name, table.Name, true)
|
||||
}
|
||||
}
|
||||
|
||||
for _, key := range order {
|
||||
tables := owners[key]
|
||||
results = append(results, createResult(
|
||||
ruleName,
|
||||
len(tables) == 1,
|
||||
rule.Message,
|
||||
formatLocation(schema.Name, display[key], "")+" on "+strings.Join(tables, ", "),
|
||||
map[string]interface{}{
|
||||
"schema": schema.Name,
|
||||
"index": display[key],
|
||||
"tables": tables,
|
||||
"occurrences": len(tables),
|
||||
},
|
||||
))
|
||||
}
|
||||
}
|
||||
|
||||
return results
|
||||
}
|
||||
|
||||
@@ -835,3 +835,84 @@ func TestFormatLocation(t *testing.T) {
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateDuplicateIndexName(t *testing.T) {
|
||||
db := &models.Database{
|
||||
Name: "testdb",
|
||||
Schemas: []*models.Schema{
|
||||
{
|
||||
Name: "entity",
|
||||
Tables: []*models.Table{
|
||||
{
|
||||
Name: "actor_phone",
|
||||
Indexes: map[string]*models.Index{
|
||||
"idx_actor": {Name: "idx_actor", Columns: []string{"rid_actor"}},
|
||||
"uk_phone": {Name: "uk_phone", Columns: []string{"phone"}, Unique: true},
|
||||
},
|
||||
Constraints: map[string]*models.Constraint{
|
||||
// Same name as the index on the same table: one object.
|
||||
"uk_phone": {Name: "uk_phone", Type: models.UniqueConstraint, Columns: []string{"phone"}},
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: "actor_email",
|
||||
Indexes: map[string]*models.Index{
|
||||
"idx_actor": {Name: "idx_actor", Columns: []string{"rid_actor"}},
|
||||
},
|
||||
Constraints: map[string]*models.Constraint{
|
||||
"UK_Phone": {Name: "UK_Phone", Type: models.UniqueConstraint, Columns: []string{"email"}},
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: "actor_address",
|
||||
Indexes: map[string]*models.Index{
|
||||
"idx_actor": {Name: "idx_actor", Columns: []string{"rid_actor"}},
|
||||
"idx_actor#2": {Name: "idx_actor", Columns: []string{"rid_actor", "kind"}},
|
||||
"idx_address": {Name: "idx_address", Columns: []string{"line1"}},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
// Same names in another schema do not collide.
|
||||
Name: "org",
|
||||
Tables: []*models.Table{
|
||||
{Name: "api_provider", Indexes: map[string]*models.Index{"idx_actor": {Name: "idx_actor"}}},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
results := validateDuplicateIndexName(db, Rule{Message: "dup"}, "duplicate_index_name")
|
||||
|
||||
got := map[string]bool{}
|
||||
occ := map[string]int{}
|
||||
for _, r := range results {
|
||||
key := r.Context["schema"].(string) + "." + r.Context["index"].(string)
|
||||
got[key] = r.Passed
|
||||
occ[key] = r.Context["occurrences"].(int)
|
||||
}
|
||||
|
||||
want := map[string]struct {
|
||||
passed bool
|
||||
occ int
|
||||
}{
|
||||
"entity.idx_actor": {false, 4},
|
||||
"entity.uk_phone": {false, 2},
|
||||
"entity.idx_address": {true, 1},
|
||||
"org.idx_actor": {true, 1},
|
||||
}
|
||||
if len(got) != len(want) {
|
||||
t.Fatalf("got %d results %v, want %d", len(got), got, len(want))
|
||||
}
|
||||
for k, w := range want {
|
||||
p, ok := got[k]
|
||||
if !ok {
|
||||
t.Errorf("missing result for %s", k)
|
||||
continue
|
||||
}
|
||||
if p != w.passed || occ[k] != w.occ {
|
||||
t.Errorf("%s: passed=%v occurrences=%d, want passed=%v occurrences=%d", k, p, occ[k], w.passed, w.occ)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+500
-30
@@ -19,24 +19,54 @@ import (
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"gopkg.in/yaml.v3"
|
||||
)
|
||||
|
||||
// SchemaVersion is the only job-file schema version this build understands.
|
||||
const SchemaVersion = 1
|
||||
// CurrentSchemaVersion is the highest job-file schema version this build was
|
||||
// written for. MinSchemaVersion is the oldest it still accepts. A file that
|
||||
// declares a version in between loads normally; a newer version loads
|
||||
// best-effort with a warning (see Load); an older-than-minimum version is a
|
||||
// hard error.
|
||||
const (
|
||||
CurrentSchemaVersion = 1
|
||||
MinSchemaVersion = 1
|
||||
)
|
||||
|
||||
// Built-in logfile rotation policy, used when neither the job nor its file's
|
||||
// defaults block sets one.
|
||||
const (
|
||||
defaultLogMaxSizeBytes int64 = 5 << 20 // 5 MiB
|
||||
defaultLogKeep = 3
|
||||
)
|
||||
|
||||
// Command names are a closed allow-list. Arbitrary strings are rejected.
|
||||
const (
|
||||
CommandConvert = "convert" // read one or more schema files, optionally merge, write one output
|
||||
CommandMerge = "merge" // additive merge of two or more schema files into one output
|
||||
CommandScriptsList = "scripts-list" // deterministically list SQL scripts across one or more directories
|
||||
CommandScriptsExec = "scripts-exec" // execute SQL scripts across one or more directories against a live database
|
||||
CommandTempl = "templ" // apply a custom Go text template to one or more schemas
|
||||
CommandSplit = "split" // extract selected schemas/tables into a separate output
|
||||
CommandInspect = "inspect" // validate one or more schemas against rules and write a report
|
||||
CommandDiff = "diff" // compare exactly two schemas and write a differences report
|
||||
)
|
||||
|
||||
// SupportedCommands lists every accepted command, in help order.
|
||||
var SupportedCommands = []string{CommandConvert, CommandMerge, CommandScriptsList}
|
||||
var SupportedCommands = []string{
|
||||
CommandConvert, CommandMerge, CommandScriptsList, CommandScriptsExec,
|
||||
CommandTempl, CommandSplit, CommandInspect, CommandDiff,
|
||||
}
|
||||
|
||||
// producerCommands are commands whose output is a schema file that another job
|
||||
// may consume via from_job.
|
||||
var producerCommands = map[string]bool{
|
||||
CommandConvert: true, CommandMerge: true, CommandSplit: true,
|
||||
}
|
||||
|
||||
// readerFormats are the file-based input formats a job may declare (path).
|
||||
var readerFormats = map[string]bool{
|
||||
@@ -60,10 +90,41 @@ var writerFormats = map[string]bool{
|
||||
// live database) is supported instead of writing a file.
|
||||
var execOutputFormats = map[string]bool{"pgsql": true}
|
||||
|
||||
// singleFileFormats are output formats that emit exactly one file (as opposed
|
||||
// to a directory of files). Only these are eligible for atomic temp+rename
|
||||
// writes and for being consumed by another job via from_job.
|
||||
var singleFileFormats = map[string]bool{
|
||||
"json": true, "yaml": true, "dbml": true, "dctx": true, "drawdb": true,
|
||||
"graphql": true, "pgsql": true, "mssql": true, "sqlite": true,
|
||||
}
|
||||
|
||||
// SingleFileOutputFormat reports whether format writes exactly one file.
|
||||
func SingleFileOutputFormat(format string) bool {
|
||||
return singleFileFormats[strings.ToLower(format)]
|
||||
}
|
||||
|
||||
// diffReportFormats and inspectReportFormats are the report.format values
|
||||
// accepted by the diff and inspect commands respectively.
|
||||
var (
|
||||
diffReportFormats = map[string]bool{"summary": true, "json": true, "html": true}
|
||||
inspectReportFormats = map[string]bool{"markdown": true, "json": true}
|
||||
)
|
||||
|
||||
// File is the on-disk shape of a single job file.
|
||||
type File struct {
|
||||
Version int `yaml:"version"`
|
||||
Jobs map[string]*Job `yaml:"jobs"`
|
||||
Version int `yaml:"version"`
|
||||
Defaults *Defaults `yaml:"defaults"`
|
||||
Jobs map[string]*Job `yaml:"jobs"`
|
||||
}
|
||||
|
||||
// Defaults carries file-wide settings that individual jobs may override.
|
||||
type Defaults struct {
|
||||
// LogMaxSize is a human-readable size ("5MB", "512KB", "1GB"). Empty
|
||||
// means "use the built-in default".
|
||||
LogMaxSize string `yaml:"log_max_size"`
|
||||
// LogKeep is how many rotated logfiles to retain. Zero means "use the
|
||||
// built-in default".
|
||||
LogKeep int `yaml:"log_keep"`
|
||||
}
|
||||
|
||||
// Job is one named job within a job file.
|
||||
@@ -71,15 +132,26 @@ type Job struct {
|
||||
// Name and SourceFile are populated by Load, not parsed from YAML.
|
||||
Name string `yaml:"-"`
|
||||
SourceFile string `yaml:"-"`
|
||||
// fileDefaults is the Defaults block of the file that declared this job,
|
||||
// captured by Load. nil when the file had none.
|
||||
fileDefaults *Defaults `yaml:"-"`
|
||||
|
||||
Command string `yaml:"command"`
|
||||
Description string `yaml:"description"`
|
||||
DependsOn []string `yaml:"depends_on"`
|
||||
Inputs []Input `yaml:"inputs"`
|
||||
ScriptDirs []string `yaml:"script_dirs"`
|
||||
Output *Output `yaml:"output"`
|
||||
Options Options `yaml:"options"`
|
||||
Logfile string `yaml:"logfile"`
|
||||
Command string `yaml:"command"`
|
||||
Description string `yaml:"description"`
|
||||
DependsOn []string `yaml:"depends_on"`
|
||||
Inputs []Input `yaml:"inputs"`
|
||||
ScriptDirs []string `yaml:"script_dirs"`
|
||||
Template string `yaml:"template"`
|
||||
Mode string `yaml:"mode"`
|
||||
FilenamePattern string `yaml:"filename_pattern"`
|
||||
Output *Output `yaml:"output"`
|
||||
Rules string `yaml:"rules"`
|
||||
Report *Report `yaml:"report"`
|
||||
Select *Select `yaml:"select"`
|
||||
Options Options `yaml:"options"`
|
||||
Logfile string `yaml:"logfile"`
|
||||
LogMaxSize string `yaml:"log_max_size"`
|
||||
LogKeep *int `yaml:"log_keep"`
|
||||
}
|
||||
|
||||
// Input is one declared input schema.
|
||||
@@ -90,6 +162,10 @@ type Input struct {
|
||||
// ConnEnv is the NAME of an environment variable holding a connection
|
||||
// string, used with database formats. The value is never stored here.
|
||||
ConnEnv string `yaml:"conn_env"`
|
||||
// FromJob names another job in the set whose file output is used as this
|
||||
// input. It implies a dependency on that job. Path/Format/ConnEnv must be
|
||||
// empty when FromJob is set; the format is inherited from the producer.
|
||||
FromJob string `yaml:"from_job"`
|
||||
}
|
||||
|
||||
// Output is the declared output target.
|
||||
@@ -100,17 +176,123 @@ type Output struct {
|
||||
Overwrite bool `yaml:"overwrite"`
|
||||
}
|
||||
|
||||
// Report is the output target for the inspect and diff commands.
|
||||
type Report struct {
|
||||
// Format is the report format: diff accepts summary|json|html, inspect
|
||||
// accepts markdown|json. Empty means the command's default.
|
||||
Format string `yaml:"format"`
|
||||
Path string `yaml:"path"`
|
||||
Overwrite bool `yaml:"overwrite"`
|
||||
}
|
||||
|
||||
// Select carries the schema/table selection for the split command.
|
||||
type Select struct {
|
||||
Schemas []string `yaml:"schemas"`
|
||||
Tables []string `yaml:"tables"`
|
||||
ExcludeSchemas []string `yaml:"exclude_schemas"`
|
||||
ExcludeTables []string `yaml:"exclude_tables"`
|
||||
DatabaseName string `yaml:"database_name"`
|
||||
}
|
||||
|
||||
// LogPolicy is the resolved logfile rotation policy for a job.
|
||||
type LogPolicy struct {
|
||||
MaxSizeBytes int64
|
||||
Keep int
|
||||
}
|
||||
|
||||
// ResolvedLogPolicy returns the effective rotation policy: the job's own
|
||||
// overrides win, then its file's defaults block, then the built-in default.
|
||||
func (j *Job) ResolvedLogPolicy() LogPolicy {
|
||||
p := LogPolicy{MaxSizeBytes: defaultLogMaxSizeBytes, Keep: defaultLogKeep}
|
||||
|
||||
if j.fileDefaults != nil {
|
||||
if n, err := parseHumanSize(j.fileDefaults.LogMaxSize); err == nil && n > 0 {
|
||||
p.MaxSizeBytes = n
|
||||
}
|
||||
if j.fileDefaults.LogKeep > 0 {
|
||||
p.Keep = j.fileDefaults.LogKeep
|
||||
}
|
||||
}
|
||||
if n, err := parseHumanSize(j.LogMaxSize); err == nil && n > 0 {
|
||||
p.MaxSizeBytes = n
|
||||
}
|
||||
if j.LogKeep != nil && *j.LogKeep >= 0 {
|
||||
p.Keep = *j.LogKeep
|
||||
}
|
||||
return p
|
||||
}
|
||||
|
||||
// effectiveDeps returns the union of explicit depends_on entries and the jobs
|
||||
// referenced by from_job inputs, deduplicated in stable order.
|
||||
func (j *Job) effectiveDeps() []string {
|
||||
seen := map[string]bool{}
|
||||
var deps []string
|
||||
add := func(name string) {
|
||||
if name == "" || name == j.Name || seen[name] {
|
||||
return
|
||||
}
|
||||
seen[name] = true
|
||||
deps = append(deps, name)
|
||||
}
|
||||
for _, d := range j.DependsOn {
|
||||
add(d)
|
||||
}
|
||||
for _, in := range j.Inputs {
|
||||
add(in.FromJob)
|
||||
}
|
||||
return deps
|
||||
}
|
||||
|
||||
// parseHumanSize parses a byte size such as "5MB", "512 KB", "1gb" or a bare
|
||||
// byte count. An empty string returns (0, nil) so callers can fall back.
|
||||
func parseHumanSize(s string) (int64, error) {
|
||||
s = strings.TrimSpace(s)
|
||||
if s == "" {
|
||||
return 0, nil
|
||||
}
|
||||
upper := strings.ToUpper(s)
|
||||
mult := int64(1)
|
||||
// Check multi-character suffixes before the bare "B".
|
||||
for _, u := range []struct {
|
||||
suffix string
|
||||
m int64
|
||||
}{
|
||||
{"KB", 1 << 10}, {"MB", 1 << 20}, {"GB", 1 << 30}, {"B", 1},
|
||||
} {
|
||||
if strings.HasSuffix(upper, u.suffix) {
|
||||
mult = u.m
|
||||
upper = strings.TrimSpace(strings.TrimSuffix(upper, u.suffix))
|
||||
break
|
||||
}
|
||||
}
|
||||
n, err := strconv.ParseFloat(upper, 64)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("invalid size %q", s)
|
||||
}
|
||||
if n < 0 {
|
||||
return 0, fmt.Errorf("negative size %q", s)
|
||||
}
|
||||
return int64(n * float64(mult)), nil
|
||||
}
|
||||
|
||||
// Options carries the subset of command flags a job file may set.
|
||||
type Options struct {
|
||||
FlattenSchema bool `yaml:"flatten_schema"`
|
||||
Schema string `yaml:"schema"`
|
||||
Package string `yaml:"package"`
|
||||
FlattenSchema bool `yaml:"flatten_schema"`
|
||||
Schema string `yaml:"schema"`
|
||||
Package string `yaml:"package"`
|
||||
// Types and ArrayNullable mirror the existing convert/split writer flags.
|
||||
// They are passed through only to writers that already support them.
|
||||
Types string `yaml:"types"`
|
||||
ArrayNullable string `yaml:"array_nullable"`
|
||||
ContinueOnError bool `yaml:"continue_on_error"`
|
||||
SkipRelations bool `yaml:"skip_relations"`
|
||||
SkipEnums bool `yaml:"skip_enums"`
|
||||
SkipViews bool `yaml:"skip_views"`
|
||||
SkipDomains bool `yaml:"skip_domains"`
|
||||
SkipSequences bool `yaml:"skip_sequences"`
|
||||
// FullDDL makes direct pgsql output execute the full idempotent DDL instead of
|
||||
// diffing against the live database first.
|
||||
FullDDL bool `yaml:"full_ddl"`
|
||||
SkipRelations bool `yaml:"skip_relations"`
|
||||
SkipEnums bool `yaml:"skip_enums"`
|
||||
SkipViews bool `yaml:"skip_views"`
|
||||
SkipDomains bool `yaml:"skip_domains"`
|
||||
SkipSequences bool `yaml:"skip_sequences"`
|
||||
}
|
||||
|
||||
// Dir returns the directory that a job's relative paths resolve against:
|
||||
@@ -123,6 +305,9 @@ type Set struct {
|
||||
Files []string
|
||||
// Jobs is keyed by job name.
|
||||
Jobs map[string]*Job
|
||||
// Warnings holds non-fatal load-time messages (e.g. a newer-than-known
|
||||
// schema version). Callers should surface these to the user.
|
||||
Warnings []string
|
||||
}
|
||||
|
||||
// Names returns all job names in deterministic (sorted) order.
|
||||
@@ -195,15 +380,35 @@ func Load(paths []string) (*Set, error) {
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to read job file %q: %w", path, err)
|
||||
}
|
||||
|
||||
// Peek at the version first so a newer file can be parsed leniently
|
||||
// (unknown fields ignored) instead of failing outright.
|
||||
var probe struct {
|
||||
Version int `yaml:"version"`
|
||||
}
|
||||
if err := yaml.Unmarshal(data, &probe); err != nil {
|
||||
return nil, fmt.Errorf("invalid job file %q: %w", path, err)
|
||||
}
|
||||
version := probe.Version
|
||||
if version == 0 {
|
||||
version = CurrentSchemaVersion
|
||||
}
|
||||
if version < MinSchemaVersion {
|
||||
return nil, fmt.Errorf("job file %q: unsupported version %d (this build accepts %d or newer)", path, version, MinSchemaVersion)
|
||||
}
|
||||
strict := version <= CurrentSchemaVersion
|
||||
if !strict {
|
||||
set.Warnings = append(set.Warnings, fmt.Sprintf(
|
||||
"job file %q declares version %d, newer than this build understands (%d); loading best-effort and ignoring unknown fields",
|
||||
path, version, CurrentSchemaVersion))
|
||||
}
|
||||
|
||||
dec := yaml.NewDecoder(strings.NewReader(string(data)))
|
||||
dec.KnownFields(true)
|
||||
dec.KnownFields(strict)
|
||||
var f File
|
||||
if err := dec.Decode(&f); err != nil {
|
||||
return nil, fmt.Errorf("invalid job file %q: %w", path, err)
|
||||
}
|
||||
if f.Version != SchemaVersion {
|
||||
return nil, fmt.Errorf("job file %q: unsupported version %d (expected %d)", path, f.Version, SchemaVersion)
|
||||
}
|
||||
if len(f.Jobs) == 0 {
|
||||
return nil, fmt.Errorf("job file %q: no jobs defined", path)
|
||||
}
|
||||
@@ -216,6 +421,7 @@ func Load(paths []string) (*Set, error) {
|
||||
}
|
||||
job.Name = name
|
||||
job.SourceFile = path
|
||||
job.fileDefaults = f.Defaults
|
||||
origin[name] = path
|
||||
set.Jobs[name] = job
|
||||
}
|
||||
@@ -236,13 +442,30 @@ func (s *Set) Validate() error {
|
||||
errs = append(errs, fmt.Sprintf("job %q: %s", name, msg))
|
||||
}
|
||||
}
|
||||
// Dependency references + cycles.
|
||||
// Dependency references + cycles + from_job wiring.
|
||||
for _, name := range s.Names() {
|
||||
for _, dep := range s.Jobs[name].DependsOn {
|
||||
j := s.Jobs[name]
|
||||
for _, dep := range j.DependsOn {
|
||||
if _, ok := s.Jobs[dep]; !ok {
|
||||
errs = append(errs, fmt.Sprintf("job %q: depends_on unknown job %q", name, dep))
|
||||
}
|
||||
}
|
||||
for i, in := range j.Inputs {
|
||||
if in.FromJob == "" {
|
||||
continue
|
||||
}
|
||||
producer, ok := s.Jobs[in.FromJob]
|
||||
if !ok {
|
||||
errs = append(errs, fmt.Sprintf("job %q: input[%d] from_job references unknown job %q", name, i, in.FromJob))
|
||||
continue
|
||||
}
|
||||
if !producerCommands[producer.Command] || producer.Output == nil ||
|
||||
producer.Output.Path == "" || !SingleFileOutputFormat(producer.Output.Format) {
|
||||
errs = append(errs, fmt.Sprintf(
|
||||
"job %q: input[%d] from_job %q must name a convert/merge/split job that writes a single-file output",
|
||||
name, i, in.FromJob))
|
||||
}
|
||||
}
|
||||
}
|
||||
if cycle := s.findCycle(); cycle != "" {
|
||||
errs = append(errs, fmt.Sprintf("dependency cycle detected: %s", cycle))
|
||||
@@ -258,7 +481,8 @@ func (j *Job) validate() []string {
|
||||
var e []string
|
||||
|
||||
switch j.Command {
|
||||
case CommandConvert, CommandMerge, CommandScriptsList:
|
||||
case CommandConvert, CommandMerge, CommandScriptsList, CommandScriptsExec,
|
||||
CommandTempl, CommandSplit, CommandInspect, CommandDiff:
|
||||
case "":
|
||||
e = append(e, "missing command")
|
||||
return e
|
||||
@@ -277,6 +501,8 @@ func (j *Job) validate() []string {
|
||||
}
|
||||
}
|
||||
checkPath("logfile", j.Logfile)
|
||||
checkPath("template", j.Template)
|
||||
checkPath("rules", j.Rules)
|
||||
for _, in := range j.Inputs {
|
||||
checkPath("input path", in.Path)
|
||||
}
|
||||
@@ -286,6 +512,13 @@ func (j *Job) validate() []string {
|
||||
if j.Output != nil {
|
||||
checkPath("output path", j.Output.Path)
|
||||
}
|
||||
if j.Report != nil {
|
||||
checkPath("report path", j.Report.Path)
|
||||
}
|
||||
|
||||
if _, err := parseHumanSize(j.LogMaxSize); err != nil {
|
||||
e = append(e, fmt.Sprintf("log_max_size: %v", err))
|
||||
}
|
||||
|
||||
switch j.Command {
|
||||
case CommandConvert, CommandMerge:
|
||||
@@ -317,11 +550,192 @@ func (j *Job) validate() []string {
|
||||
if j.Output != nil {
|
||||
e = append(e, "output is not valid for command \"scripts-list\"")
|
||||
}
|
||||
case CommandTempl:
|
||||
if len(j.Inputs) < 1 {
|
||||
e = append(e, "command \"templ\" requires at least 1 input")
|
||||
}
|
||||
for i, in := range j.Inputs {
|
||||
e = append(e, validateTemplInput(i, in)...)
|
||||
}
|
||||
if j.Template == "" {
|
||||
e = append(e, "command \"templ\" requires template")
|
||||
}
|
||||
mode := strings.ToLower(j.Mode)
|
||||
if mode == "" {
|
||||
mode = "database"
|
||||
}
|
||||
switch mode {
|
||||
case "database", "schema", "script", "table":
|
||||
default:
|
||||
e = append(e, fmt.Sprintf("command \"templ\" has unsupported mode %q (supported: database, schema, script, table)", j.Mode))
|
||||
}
|
||||
if len(j.ScriptDirs) > 0 {
|
||||
e = append(e, "script_dirs is not valid for command \"templ\"")
|
||||
}
|
||||
if j.Output != nil && j.Output.ConnEnv != "" {
|
||||
e = append(e, "command \"templ\" does not support database output")
|
||||
}
|
||||
if j.Output != nil && j.Output.Format != "" {
|
||||
if !strings.EqualFold(j.Output.Format, "text") {
|
||||
e = append(e, "command \"templ\" accepts only output.format: text")
|
||||
}
|
||||
}
|
||||
case CommandSplit:
|
||||
if len(j.Inputs) < 1 {
|
||||
e = append(e, "command \"split\" requires at least 1 input")
|
||||
}
|
||||
for i, in := range j.Inputs {
|
||||
e = append(e, validateInput(i, in)...)
|
||||
}
|
||||
if len(j.ScriptDirs) > 0 {
|
||||
e = append(e, "script_dirs is not valid for command \"split\"")
|
||||
}
|
||||
if j.Report != nil {
|
||||
e = append(e, "report is not valid for command \"split\" (use output)")
|
||||
}
|
||||
if j.Output == nil {
|
||||
e = append(e, "missing output")
|
||||
} else {
|
||||
if j.Output.ConnEnv != "" {
|
||||
e = append(e, "command \"split\" writes a file; output.conn_env is not supported")
|
||||
}
|
||||
e = append(e, validateOutput(*j.Output)...)
|
||||
}
|
||||
case CommandInspect:
|
||||
if len(j.Inputs) < 1 {
|
||||
e = append(e, "command \"inspect\" requires at least 1 input")
|
||||
}
|
||||
for i, in := range j.Inputs {
|
||||
e = append(e, validateInput(i, in)...)
|
||||
}
|
||||
if len(j.ScriptDirs) > 0 {
|
||||
e = append(e, "script_dirs is not valid for command \"inspect\"")
|
||||
}
|
||||
if j.Output != nil {
|
||||
e = append(e, "output is not valid for command \"inspect\" (use report)")
|
||||
}
|
||||
e = append(e, validateReport(j.Report, "inspect", inspectReportFormats, "markdown")...)
|
||||
case CommandDiff:
|
||||
if len(j.Inputs) != 2 {
|
||||
e = append(e, "command \"diff\" requires exactly 2 inputs (source, target)")
|
||||
}
|
||||
for i, in := range j.Inputs {
|
||||
e = append(e, validateInput(i, in)...)
|
||||
}
|
||||
if len(j.ScriptDirs) > 0 {
|
||||
e = append(e, "script_dirs is not valid for command \"diff\"")
|
||||
}
|
||||
if j.Output != nil {
|
||||
e = append(e, "output is not valid for command \"diff\" (use report)")
|
||||
}
|
||||
e = append(e, validateReport(j.Report, "diff", diffReportFormats, "summary")...)
|
||||
case CommandScriptsExec:
|
||||
if len(j.ScriptDirs) == 0 {
|
||||
e = append(e, "command \"scripts-exec\" requires at least one script_dir")
|
||||
}
|
||||
if len(j.Inputs) > 0 {
|
||||
e = append(e, "inputs is not valid for command \"scripts-exec\"")
|
||||
}
|
||||
if j.Report != nil {
|
||||
e = append(e, "report is not valid for command \"scripts-exec\"")
|
||||
}
|
||||
if j.Output == nil || j.Output.ConnEnv == "" {
|
||||
e = append(e, "command \"scripts-exec\" requires output.conn_env (an environment variable name holding a connection string)")
|
||||
} else {
|
||||
if j.Output.Path != "" {
|
||||
e = append(e, "command \"scripts-exec\" executes against a database; output.path is not supported")
|
||||
}
|
||||
f := strings.ToLower(j.Output.Format)
|
||||
if f != "" && f != "pgsql" {
|
||||
e = append(e, fmt.Sprintf("command \"scripts-exec\" only supports pgsql databases (got %q)", j.Output.Format))
|
||||
}
|
||||
if looksLikeSecret(j.Output.ConnEnv) {
|
||||
e = append(e, "output: conn_env must be an environment variable name, not a connection string")
|
||||
}
|
||||
}
|
||||
}
|
||||
return e
|
||||
}
|
||||
|
||||
// validateReport checks a Report block for the inspect/diff commands.
|
||||
func validateReport(r *Report, cmd string, allowed map[string]bool, defFmt string) []string {
|
||||
if r == nil {
|
||||
return []string{fmt.Sprintf("command %q requires a report block", cmd)}
|
||||
}
|
||||
var e []string
|
||||
f := strings.ToLower(r.Format)
|
||||
if f == "" {
|
||||
f = defFmt
|
||||
}
|
||||
if !allowed[f] {
|
||||
names := make([]string, 0, len(allowed))
|
||||
for k := range allowed {
|
||||
names = append(names, k)
|
||||
}
|
||||
sort.Strings(names)
|
||||
e = append(e, fmt.Sprintf("command %q report.format %q is not supported (use: %s)", cmd, r.Format, strings.Join(names, ", ")))
|
||||
}
|
||||
// A diff summary may be written to the log; everything else needs a path.
|
||||
summaryToLog := cmd == "diff" && f == "summary"
|
||||
if r.Path == "" && !summaryToLog {
|
||||
e = append(e, fmt.Sprintf("command %q requires report.path", cmd))
|
||||
}
|
||||
return e
|
||||
}
|
||||
|
||||
func validateTemplInput(i int, in Input) []string {
|
||||
if in.FromJob != "" {
|
||||
return fromJobInputShape(i, in)
|
||||
}
|
||||
var e []string
|
||||
if in.Format == "" {
|
||||
return []string{fmt.Sprintf("input[%d]: missing format", i)}
|
||||
}
|
||||
f := strings.ToLower(in.Format)
|
||||
if f == "pgsql" {
|
||||
if in.ConnEnv == "" {
|
||||
e = append(e, fmt.Sprintf("input[%d]: format %q requires conn_env (an environment variable name)", i, in.Format))
|
||||
}
|
||||
if in.Path != "" {
|
||||
e = append(e, fmt.Sprintf("input[%d]: format %q takes conn_env, not path", i, in.Format))
|
||||
}
|
||||
} else if readerFormats[f] {
|
||||
if in.Path == "" {
|
||||
e = append(e, fmt.Sprintf("input[%d]: missing path", i))
|
||||
}
|
||||
if in.ConnEnv != "" {
|
||||
e = append(e, fmt.Sprintf("input[%d]: format %q does not use conn_env", i, in.Format))
|
||||
}
|
||||
} else {
|
||||
e = append(e, fmt.Sprintf("input[%d]: unsupported templ input format %q", i, in.Format))
|
||||
}
|
||||
if looksLikeSecret(in.ConnEnv) {
|
||||
e = append(e, fmt.Sprintf("input[%d]: conn_env must be an environment variable name, not a connection string", i))
|
||||
}
|
||||
return e
|
||||
}
|
||||
|
||||
// fromJobInputShape checks the structural rules for an input that pulls its
|
||||
// schema from another job's output. The referenced job's existence and kind
|
||||
// are checked in Set.Validate, which can see the whole set.
|
||||
func fromJobInputShape(i int, in Input) []string {
|
||||
var e []string
|
||||
if in.Path != "" {
|
||||
e = append(e, fmt.Sprintf("input[%d]: from_job takes no path", i))
|
||||
}
|
||||
if in.Format != "" {
|
||||
e = append(e, fmt.Sprintf("input[%d]: from_job inherits the producer's format; drop format", i))
|
||||
}
|
||||
if in.ConnEnv != "" {
|
||||
e = append(e, fmt.Sprintf("input[%d]: from_job takes no conn_env", i))
|
||||
}
|
||||
return e
|
||||
}
|
||||
|
||||
func validateInput(i int, in Input) []string {
|
||||
if in.FromJob != "" {
|
||||
return fromJobInputShape(i, in)
|
||||
}
|
||||
var e []string
|
||||
if in.Format == "" {
|
||||
e = append(e, fmt.Sprintf("input[%d]: missing format", i))
|
||||
@@ -425,9 +839,65 @@ func SafeJoin(root, rel string) (string, error) {
|
||||
if rp == ".." || strings.HasPrefix(rp, ".."+string(filepath.Separator)) {
|
||||
return "", fmt.Errorf("path %q escapes the job file directory", rel)
|
||||
}
|
||||
// Symlink hardening: resolve symlinks on the root and on the deepest
|
||||
// existing ancestor of the target, and require the target to still live
|
||||
// inside the resolved root. This catches a symlink inside the job-file
|
||||
// directory that points outside it.
|
||||
realRoot, err := filepath.EvalSymlinks(absRoot)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("cannot resolve job file directory: %w", err)
|
||||
}
|
||||
realAnc, err := filepath.EvalSymlinks(deepestExistingAncestor(joined))
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("cannot resolve path %q: %w", rel, err)
|
||||
}
|
||||
if realAnc != realRoot {
|
||||
if r, err := filepath.Rel(realRoot, realAnc); err != nil ||
|
||||
r == ".." || strings.HasPrefix(r, ".."+string(filepath.Separator)) {
|
||||
return "", fmt.Errorf("path %q resolves outside the job file directory via a symlink", rel)
|
||||
}
|
||||
}
|
||||
return joined, nil
|
||||
}
|
||||
|
||||
var envReferencePattern = regexp.MustCompile(`\$\{([A-Za-z_][A-Za-z0-9_]*)\}`)
|
||||
|
||||
// ExpandEnv expands ${NAME} references using the current process environment.
|
||||
// It deliberately supports only shell-independent variable references; job
|
||||
// files are never passed through a shell. An unset variable is an error so a
|
||||
// typo cannot silently turn into a relative path.
|
||||
func ExpandEnv(value string) (string, error) {
|
||||
var missing string
|
||||
expanded := envReferencePattern.ReplaceAllStringFunc(value, func(reference string) string {
|
||||
name := reference[2 : len(reference)-1]
|
||||
resolved, ok := os.LookupEnv(name)
|
||||
if !ok {
|
||||
missing = name
|
||||
return reference
|
||||
}
|
||||
return resolved
|
||||
})
|
||||
if missing != "" {
|
||||
return "", fmt.Errorf("environment variable %q referenced by ${%s} is not set", missing, missing)
|
||||
}
|
||||
return expanded, nil
|
||||
}
|
||||
|
||||
// deepestExistingAncestor returns p itself if it exists, otherwise the nearest
|
||||
// existing parent directory (falling back to the filesystem root).
|
||||
func deepestExistingAncestor(p string) string {
|
||||
for {
|
||||
if _, err := os.Lstat(p); err == nil {
|
||||
return p
|
||||
}
|
||||
parent := filepath.Dir(p)
|
||||
if parent == p {
|
||||
return p
|
||||
}
|
||||
p = parent
|
||||
}
|
||||
}
|
||||
|
||||
// Plan returns the jobs to execute for name in dependency order. When
|
||||
// includeDeps is false only the named job is returned (its declared
|
||||
// dependencies are still validated to exist and be acyclic by Validate).
|
||||
@@ -452,7 +922,7 @@ func (s *Set) Plan(name string, includeDeps bool) ([]*Job, error) {
|
||||
}
|
||||
inProgress[n] = true
|
||||
j := s.Jobs[n]
|
||||
deps := append([]string(nil), j.DependsOn...)
|
||||
deps := j.effectiveDeps()
|
||||
sort.Strings(deps)
|
||||
for _, d := range deps {
|
||||
if _, ok := s.Jobs[d]; !ok {
|
||||
@@ -481,7 +951,7 @@ func (s *Set) findCycle() string {
|
||||
dfs = func(n string) []string {
|
||||
color[n] = 1
|
||||
stack = append(stack, n)
|
||||
deps := append([]string(nil), s.Jobs[n].DependsOn...)
|
||||
deps := s.Jobs[n].effectiveDeps()
|
||||
sort.Strings(deps)
|
||||
for _, d := range deps {
|
||||
if _, ok := s.Jobs[d]; !ok {
|
||||
|
||||
+218
-5
@@ -56,13 +56,97 @@ func TestLoadRejectsUnknownFields(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadRejectsBadVersion(t *testing.T) {
|
||||
func TestLoadWarnsOnNewerVersion(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
p := filepath.Join(dir, "relspec.yml")
|
||||
write(t, p, "version: 2\njobs:\n a:\n command: convert\n")
|
||||
_, err := Load([]string{p})
|
||||
if err == nil || !strings.Contains(err.Error(), "unsupported version") {
|
||||
t.Fatalf("expected unsupported version error, got %v", err)
|
||||
// A newer version loads best-effort with a warning, and unknown fields
|
||||
// from the newer schema are ignored rather than rejected.
|
||||
write(t, p, "version: 99\njobs:\n a:\n command: convert\n"+
|
||||
" inputs:\n - path: a.dbml\n format: dbml\n"+
|
||||
" output:\n format: json\n path: out.json\n"+
|
||||
" future_field: whatever\n")
|
||||
set, err := Load([]string{p})
|
||||
if err != nil {
|
||||
t.Fatalf("newer version should load, got %v", err)
|
||||
}
|
||||
if len(set.Warnings) == 0 {
|
||||
t.Fatal("expected a warning about the newer version")
|
||||
}
|
||||
if err := set.Validate(); err != nil {
|
||||
t.Fatalf("validate: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadAcceptsOmittedVersion(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
p := filepath.Join(dir, "relspec.yml")
|
||||
write(t, p, "jobs:\n a:\n command: convert\n"+
|
||||
" inputs:\n - path: a.dbml\n format: dbml\n"+
|
||||
" output:\n format: json\n path: out.json\n")
|
||||
set, err := Load([]string{p})
|
||||
if err != nil {
|
||||
t.Fatalf("omitted version should load, got %v", err)
|
||||
}
|
||||
if len(set.Warnings) != 0 {
|
||||
t.Fatalf("omitted version should not warn, got %v", set.Warnings)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadStillRejectsUnknownFieldsAtCurrentVersion(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
p := filepath.Join(dir, "relspec.yml")
|
||||
write(t, p, "version: 1\njobs:\n a:\n command: convert\n bogus: true\n")
|
||||
if _, err := Load([]string{p}); err == nil {
|
||||
t.Fatal("expected unknown-field rejection at the current version")
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseHumanSize(t *testing.T) {
|
||||
cases := []struct {
|
||||
in string
|
||||
want int64
|
||||
bad bool
|
||||
}{
|
||||
{"", 0, false},
|
||||
{"512", 512, false},
|
||||
{"512B", 512, false},
|
||||
{"1KB", 1 << 10, false},
|
||||
{"5MB", 5 << 20, false},
|
||||
{"1gb", 1 << 30, false},
|
||||
{" 2 MB ", 2 << 20, false},
|
||||
{"nonsense", 0, true},
|
||||
{"-1MB", 0, true},
|
||||
}
|
||||
for _, c := range cases {
|
||||
got, err := parseHumanSize(c.in)
|
||||
if c.bad {
|
||||
if err == nil {
|
||||
t.Errorf("parseHumanSize(%q): expected error", c.in)
|
||||
}
|
||||
continue
|
||||
}
|
||||
if err != nil {
|
||||
t.Errorf("parseHumanSize(%q): %v", c.in, err)
|
||||
continue
|
||||
}
|
||||
if got != c.want {
|
||||
t.Errorf("parseHumanSize(%q) = %d, want %d", c.in, got, c.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestExpandEnv(t *testing.T) {
|
||||
t.Setenv("RELSPEC_TEST_ROOT", "build/output")
|
||||
got, err := ExpandEnv("${RELSPEC_TEST_ROOT}/schema-${RELSPEC_TEST_ROOT}")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if want := "build/output/schema-build/output"; got != want {
|
||||
t.Fatalf("ExpandEnv = %q, want %q", got, want)
|
||||
}
|
||||
|
||||
if _, err := ExpandEnv("build/${RELSPEC_TEST_MISSING}"); err == nil {
|
||||
t.Fatal("expected missing environment variable error")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -239,6 +323,135 @@ func TestShippedExampleIsValid(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestFromJobWiring(t *testing.T) {
|
||||
content := "version: 1\njobs:\n" +
|
||||
" producer:\n command: convert\n" +
|
||||
" inputs:\n - path: a.dbml\n format: dbml\n" +
|
||||
" output:\n format: json\n path: build/schema.json\n" +
|
||||
" consumer:\n command: convert\n" +
|
||||
" inputs:\n - from_job: producer\n" +
|
||||
" output:\n format: yaml\n path: build/schema.yaml\n"
|
||||
set := loadOne(t, content)
|
||||
if err := set.Validate(); err != nil {
|
||||
t.Fatalf("validate: %v", err)
|
||||
}
|
||||
plan, err := set.Plan("consumer", true)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(plan) != 2 || plan[0].Name != "producer" || plan[1].Name != "consumer" {
|
||||
t.Fatalf("plan = %v, want [producer consumer]", plan)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFromJobRejectsNonProducer(t *testing.T) {
|
||||
content := "version: 1\njobs:\n" +
|
||||
" lister:\n command: scripts-list\n script_dirs: [migrations]\n" +
|
||||
" consumer:\n command: convert\n" +
|
||||
" inputs:\n - from_job: lister\n" +
|
||||
" output:\n format: yaml\n path: out.yaml\n"
|
||||
set := loadOne(t, content)
|
||||
if err := set.Validate(); err == nil || !strings.Contains(err.Error(), "from_job") {
|
||||
t.Fatalf("want from_job producer error, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFromJobRejectsUnknownJob(t *testing.T) {
|
||||
content := "version: 1\njobs:\n" +
|
||||
" consumer:\n command: convert\n" +
|
||||
" inputs:\n - from_job: ghost\n" +
|
||||
" output:\n format: yaml\n path: out.yaml\n"
|
||||
set := loadOne(t, content)
|
||||
if err := set.Validate(); err == nil || !strings.Contains(err.Error(), "unknown job") {
|
||||
t.Fatalf("want unknown job error, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFromJobCycleDetected(t *testing.T) {
|
||||
content := "version: 1\njobs:\n" +
|
||||
" a:\n command: convert\n" +
|
||||
" inputs:\n - from_job: b\n" +
|
||||
" output:\n format: json\n path: a.json\n" +
|
||||
" b:\n command: convert\n" +
|
||||
" inputs:\n - from_job: a\n" +
|
||||
" output:\n format: json\n path: b.json\n"
|
||||
set := loadOne(t, content)
|
||||
if err := set.Validate(); err == nil || !strings.Contains(err.Error(), "cycle") {
|
||||
t.Fatalf("want cycle error, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSplitJobValidation(t *testing.T) {
|
||||
set := loadOne(t, "version: 1\njobs:\n s:\n command: split\n"+
|
||||
" inputs:\n - path: a.dbml\n format: dbml\n")
|
||||
if err := set.Validate(); err == nil || !strings.Contains(err.Error(), "missing output") {
|
||||
t.Fatalf("want missing output, got %v", err)
|
||||
}
|
||||
set = loadOne(t, "version: 1\njobs:\n s:\n command: split\n"+
|
||||
" inputs:\n - path: a.dbml\n format: dbml\n"+
|
||||
" select:\n tables: [users]\n"+
|
||||
" output:\n format: json\n path: out.json\n")
|
||||
if err := set.Validate(); err != nil {
|
||||
t.Fatalf("expected valid split job, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestInspectJobValidation(t *testing.T) {
|
||||
set := loadOne(t, "version: 1\njobs:\n i:\n command: inspect\n"+
|
||||
" inputs:\n - path: a.dbml\n format: dbml\n")
|
||||
if err := set.Validate(); err == nil || !strings.Contains(err.Error(), "report") {
|
||||
t.Fatalf("want report required, got %v", err)
|
||||
}
|
||||
set = loadOne(t, "version: 1\njobs:\n i:\n command: inspect\n"+
|
||||
" inputs:\n - path: a.dbml\n format: dbml\n"+
|
||||
" report:\n format: json\n path: build/report.json\n")
|
||||
if err := set.Validate(); err != nil {
|
||||
t.Fatalf("expected valid inspect job, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDiffJobValidation(t *testing.T) {
|
||||
set := loadOne(t, "version: 1\njobs:\n d:\n command: diff\n"+
|
||||
" inputs:\n - path: a.dbml\n format: dbml\n"+
|
||||
" report:\n format: summary\n")
|
||||
if err := set.Validate(); err == nil || !strings.Contains(err.Error(), "exactly 2 inputs") {
|
||||
t.Fatalf("want exactly 2 inputs, got %v", err)
|
||||
}
|
||||
set = loadOne(t, "version: 1\njobs:\n d:\n command: diff\n"+
|
||||
" inputs:\n - path: a.dbml\n format: dbml\n"+
|
||||
" - path: b.dbml\n format: dbml\n"+
|
||||
" report:\n format: summary\n")
|
||||
if err := set.Validate(); err != nil {
|
||||
t.Fatalf("expected valid diff job, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestScriptsExecValidation(t *testing.T) {
|
||||
set := loadOne(t, "version: 1\njobs:\n x:\n command: scripts-exec\n"+
|
||||
" script_dirs: [migrations]\n")
|
||||
if err := set.Validate(); err == nil || !strings.Contains(err.Error(), "conn_env") {
|
||||
t.Fatalf("want output.conn_env required, got %v", err)
|
||||
}
|
||||
set = loadOne(t, "version: 1\njobs:\n x:\n command: scripts-exec\n"+
|
||||
" script_dirs: [migrations]\n"+
|
||||
" output:\n conn_env: TARGET_DB_URL\n")
|
||||
if err := set.Validate(); err != nil {
|
||||
t.Fatalf("expected valid scripts-exec job, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSafeJoinRejectsSymlinkEscape(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
outside := t.TempDir()
|
||||
link := filepath.Join(root, "link")
|
||||
if err := os.Symlink(outside, link); err != nil {
|
||||
t.Skipf("symlink not supported: %v", err)
|
||||
}
|
||||
if _, err := SafeJoin(root, "link/x.sql"); err == nil {
|
||||
t.Fatal("expected rejection of a path escaping via a symlink")
|
||||
}
|
||||
}
|
||||
|
||||
func TestScriptsListValidation(t *testing.T) {
|
||||
set := loadOne(t, "version: 1\njobs:\n s:\n command: scripts-list\n")
|
||||
if err := set.Validate(); err == nil || !strings.Contains(err.Error(), "script_dir") {
|
||||
|
||||
+48
-9
@@ -91,6 +91,45 @@ func (r *MergeResult) merge(target, source *models.Database, opts *MergeOptions)
|
||||
if !opts.SkipDomains {
|
||||
r.mergeDomains(target, source)
|
||||
}
|
||||
|
||||
mergeDatabaseMetadata(target, source)
|
||||
}
|
||||
|
||||
// mergeDatabaseMetadata adds missing metadata keys and unions []string values,
|
||||
// so per-file reader state (e.g. pending DBML commented refs) survives a merge.
|
||||
func mergeDatabaseMetadata(target, source *models.Database) {
|
||||
if len(source.Metadata) == 0 {
|
||||
return
|
||||
}
|
||||
if target.Metadata == nil {
|
||||
target.Metadata = make(map[string]any, len(source.Metadata))
|
||||
}
|
||||
for key, srcVal := range source.Metadata {
|
||||
tgtVal, exists := target.Metadata[key]
|
||||
if !exists {
|
||||
if list, ok := srcVal.([]string); ok {
|
||||
srcVal = append([]string(nil), list...)
|
||||
}
|
||||
target.Metadata[key] = srcVal
|
||||
continue
|
||||
}
|
||||
tgtList, tgtOK := tgtVal.([]string)
|
||||
srcList, srcOK := srcVal.([]string)
|
||||
if !tgtOK || !srcOK {
|
||||
continue
|
||||
}
|
||||
seen := make(map[string]bool, len(tgtList))
|
||||
for _, v := range tgtList {
|
||||
seen[v] = true
|
||||
}
|
||||
for _, v := range srcList {
|
||||
if !seen[v] {
|
||||
tgtList = append(tgtList, v)
|
||||
seen[v] = true
|
||||
}
|
||||
}
|
||||
target.Metadata[key] = tgtList
|
||||
}
|
||||
}
|
||||
|
||||
func (r *MergeResult) mergeSchemaContents(target, source *models.Schema, opts *MergeOptions) {
|
||||
@@ -118,7 +157,7 @@ func (r *MergeResult) mergeSchemaContents(target, source *models.Schema, opts *M
|
||||
}
|
||||
}
|
||||
|
||||
func (r *MergeResult) mergeTables(schema *models.Schema, source *models.Schema, opts *MergeOptions) {
|
||||
func (r *MergeResult) mergeTables(schema, source *models.Schema, opts *MergeOptions) {
|
||||
// Create map of existing tables
|
||||
existingTables := make(map[string]*models.Table)
|
||||
for _, table := range schema.Tables {
|
||||
@@ -150,7 +189,7 @@ func (r *MergeResult) mergeTables(schema *models.Schema, source *models.Schema,
|
||||
}
|
||||
}
|
||||
|
||||
func (r *MergeResult) mergeColumns(table *models.Table, srcTable *models.Table) {
|
||||
func (r *MergeResult) mergeColumns(table, srcTable *models.Table) {
|
||||
// Create map of existing columns
|
||||
existingColumns := make(map[string]*models.Column)
|
||||
for colName := range table.Columns {
|
||||
@@ -185,7 +224,7 @@ func (r *MergeResult) mergeColumns(table *models.Table, srcTable *models.Table)
|
||||
}
|
||||
}
|
||||
|
||||
func (r *MergeResult) mergeConstraints(table *models.Table, srcTable *models.Table) {
|
||||
func (r *MergeResult) mergeConstraints(table, srcTable *models.Table) {
|
||||
// Initialize constraints map if nil
|
||||
if table.Constraints == nil {
|
||||
table.Constraints = make(map[string]*models.Constraint)
|
||||
@@ -208,7 +247,7 @@ func (r *MergeResult) mergeConstraints(table *models.Table, srcTable *models.Tab
|
||||
}
|
||||
}
|
||||
|
||||
func (r *MergeResult) mergeIndexes(table *models.Table, srcTable *models.Table) {
|
||||
func (r *MergeResult) mergeIndexes(table, srcTable *models.Table) {
|
||||
// Initialize indexes map if nil
|
||||
if table.Indexes == nil {
|
||||
table.Indexes = make(map[string]*models.Index)
|
||||
@@ -231,7 +270,7 @@ func (r *MergeResult) mergeIndexes(table *models.Table, srcTable *models.Table)
|
||||
}
|
||||
}
|
||||
|
||||
func (r *MergeResult) mergeViews(schema *models.Schema, source *models.Schema) {
|
||||
func (r *MergeResult) mergeViews(schema, source *models.Schema) {
|
||||
// Create map of existing views
|
||||
existingViews := make(map[string]*models.View)
|
||||
for _, view := range schema.Views {
|
||||
@@ -250,7 +289,7 @@ func (r *MergeResult) mergeViews(schema *models.Schema, source *models.Schema) {
|
||||
}
|
||||
}
|
||||
|
||||
func (r *MergeResult) mergeSequences(schema *models.Schema, source *models.Schema) {
|
||||
func (r *MergeResult) mergeSequences(schema, source *models.Schema) {
|
||||
// Create map of existing sequences
|
||||
existingSequences := make(map[string]*models.Sequence)
|
||||
for _, seq := range schema.Sequences {
|
||||
@@ -269,7 +308,7 @@ func (r *MergeResult) mergeSequences(schema *models.Schema, source *models.Schem
|
||||
}
|
||||
}
|
||||
|
||||
func (r *MergeResult) mergeEnums(schema *models.Schema, source *models.Schema) {
|
||||
func (r *MergeResult) mergeEnums(schema, source *models.Schema) {
|
||||
// Create map of existing enums
|
||||
existingEnums := make(map[string]*models.Enum)
|
||||
for _, enum := range schema.Enums {
|
||||
@@ -288,7 +327,7 @@ func (r *MergeResult) mergeEnums(schema *models.Schema, source *models.Schema) {
|
||||
}
|
||||
}
|
||||
|
||||
func (r *MergeResult) mergeRelations(schema *models.Schema, source *models.Schema) {
|
||||
func (r *MergeResult) mergeRelations(schema, source *models.Schema) {
|
||||
// Create map of existing relations
|
||||
existingRelations := make(map[string]*models.Relationship)
|
||||
for _, rel := range schema.Relations {
|
||||
@@ -306,7 +345,7 @@ func (r *MergeResult) mergeRelations(schema *models.Schema, source *models.Schem
|
||||
}
|
||||
}
|
||||
|
||||
func (r *MergeResult) mergeDomains(target *models.Database, source *models.Database) {
|
||||
func (r *MergeResult) mergeDomains(target, source *models.Database) {
|
||||
// Create map of existing domains
|
||||
existingDomains := make(map[string]*models.Domain)
|
||||
for _, domain := range target.Domains {
|
||||
|
||||
@@ -721,3 +721,32 @@ func TestComplexMerge(t *testing.T) {
|
||||
t.Error("Expected ukey_users_guid constraint to exist")
|
||||
}
|
||||
}
|
||||
|
||||
func TestMergeDatabases_Metadata(t *testing.T) {
|
||||
target := &models.Database{Metadata: map[string]any{
|
||||
"refs": []string{"a", "b"},
|
||||
"name": "target",
|
||||
}}
|
||||
source := &models.Database{Metadata: map[string]any{
|
||||
"refs": []string{"b", "c"},
|
||||
"name": "source",
|
||||
"extra": []string{"x"},
|
||||
}}
|
||||
|
||||
MergeDatabases(target, source, nil)
|
||||
|
||||
if got := target.Metadata["refs"].([]string); strings.Join(got, ",") != "a,b,c" {
|
||||
t.Errorf("refs = %v, want [a b c]", got)
|
||||
}
|
||||
if got := target.Metadata["name"]; got != "target" {
|
||||
t.Errorf("name = %v, want target (existing scalar keys are kept)", got)
|
||||
}
|
||||
extra := target.Metadata["extra"].([]string)
|
||||
if strings.Join(extra, ",") != "x" {
|
||||
t.Errorf("extra = %v, want [x]", extra)
|
||||
}
|
||||
source.Metadata["extra"].([]string)[0] = "changed"
|
||||
if extra[0] != "x" {
|
||||
t.Error("copied slice must not alias the source")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,237 @@
|
||||
package models
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// Directive is a dialect-specific instruction embedded in a source schema
|
||||
// (currently DBML) that is preserved losslessly in the intermediate model and
|
||||
// consumed only by the writer for its namespace. Directives are stored in the
|
||||
// Metadata map of the object they apply to, under DirectivesMetadataKey.
|
||||
//
|
||||
// Example DBML: `@postgres: partition by RANGE (created_at)` parses to
|
||||
// Directive{Namespace: "postgres", Key: "partition", Args: "partition by RANGE (created_at)"}.
|
||||
type Directive struct {
|
||||
// Namespace is the dialect the directive targets, e.g. "postgres" or "sqlite".
|
||||
Namespace string `json:"namespace" yaml:"namespace"`
|
||||
// Key is the lowercased first token of Args, used for duplicate detection
|
||||
// and writer dispatch.
|
||||
Key string `json:"key,omitempty" yaml:"key,omitempty"`
|
||||
// Args is the verbatim argument text following the "@namespace:" prefix.
|
||||
Args string `json:"args" yaml:"args"`
|
||||
// Line is the 1-based source line the directive was read from, when known.
|
||||
Line int `json:"line,omitempty" yaml:"line,omitempty"`
|
||||
}
|
||||
|
||||
// DirectivesMetadataKey is the Metadata map key under which the ordered list of
|
||||
// dialect directives for an object is stored.
|
||||
const DirectivesMetadataKey = "directives"
|
||||
|
||||
// DirectiveKey derives the Key for a directive from its argument text: the
|
||||
// lowercased first whitespace-delimited token.
|
||||
func DirectiveKey(args string) string {
|
||||
fields := strings.Fields(args)
|
||||
if len(fields) == 0 {
|
||||
return ""
|
||||
}
|
||||
return strings.ToLower(fields[0])
|
||||
}
|
||||
|
||||
// AddDirective appends d to the directive list stored in meta. The caller is
|
||||
// responsible for ensuring meta is non-nil (all Init* constructors allocate it).
|
||||
// If d.Key is empty it is derived from d.Args.
|
||||
func AddDirective(meta map[string]any, d Directive) {
|
||||
if meta == nil {
|
||||
return
|
||||
}
|
||||
if d.Key == "" {
|
||||
d.Key = DirectiveKey(d.Args)
|
||||
}
|
||||
existing := GetDirectives(meta)
|
||||
existing = append(existing, d)
|
||||
meta[DirectivesMetadataKey] = existing
|
||||
}
|
||||
|
||||
// GetDirectives returns the directives stored in meta, sorted deterministically
|
||||
// by (Namespace, Line, Args). It tolerates both a freshly built []Directive and
|
||||
// the []any of map[string]any produced by a JSON/YAML round-trip.
|
||||
func GetDirectives(meta map[string]any) []Directive {
|
||||
if meta == nil {
|
||||
return nil
|
||||
}
|
||||
raw, ok := meta[DirectivesMetadataKey]
|
||||
if !ok || raw == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
var out []Directive
|
||||
switch v := raw.(type) {
|
||||
case []Directive:
|
||||
out = append(out, v...)
|
||||
case []any:
|
||||
for _, item := range v {
|
||||
if d, ok := directiveFromAny(item); ok {
|
||||
out = append(out, d)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
sort.SliceStable(out, func(i, j int) bool {
|
||||
if out[i].Namespace != out[j].Namespace {
|
||||
return out[i].Namespace < out[j].Namespace
|
||||
}
|
||||
if out[i].Line != out[j].Line {
|
||||
return out[i].Line < out[j].Line
|
||||
}
|
||||
return out[i].Args < out[j].Args
|
||||
})
|
||||
return out
|
||||
}
|
||||
|
||||
// directiveFromAny decodes a single directive from the loosely typed forms that
|
||||
// survive a JSON or YAML round-trip (map[string]any / map[any]any).
|
||||
func directiveFromAny(item any) (Directive, bool) {
|
||||
switch m := item.(type) {
|
||||
case Directive:
|
||||
return m, true
|
||||
case map[string]any:
|
||||
return directiveFromStringMap(m), true
|
||||
case map[any]any:
|
||||
sm := make(map[string]any, len(m))
|
||||
for k, val := range m {
|
||||
if ks, ok := k.(string); ok {
|
||||
sm[ks] = val
|
||||
}
|
||||
}
|
||||
return directiveFromStringMap(sm), true
|
||||
}
|
||||
return Directive{}, false
|
||||
}
|
||||
|
||||
func directiveFromStringMap(m map[string]any) Directive {
|
||||
d := Directive{}
|
||||
if s, ok := m["namespace"].(string); ok {
|
||||
d.Namespace = s
|
||||
}
|
||||
if s, ok := m["key"].(string); ok {
|
||||
d.Key = s
|
||||
}
|
||||
if s, ok := m["args"].(string); ok {
|
||||
d.Args = s
|
||||
}
|
||||
switch n := m["line"].(type) {
|
||||
case int:
|
||||
d.Line = n
|
||||
case int64:
|
||||
d.Line = int(n)
|
||||
case float64:
|
||||
d.Line = int(n)
|
||||
}
|
||||
if d.Key == "" {
|
||||
d.Key = DirectiveKey(d.Args)
|
||||
}
|
||||
return d
|
||||
}
|
||||
|
||||
// DirectivesForNamespace returns the directives in meta that target ns, in the
|
||||
// deterministic order of GetDirectives.
|
||||
func DirectivesForNamespace(meta map[string]any, ns string) []Directive {
|
||||
all := GetDirectives(meta)
|
||||
if len(all) == 0 {
|
||||
return nil
|
||||
}
|
||||
out := make([]Directive, 0, len(all))
|
||||
for _, d := range all {
|
||||
if d.Namespace == ns {
|
||||
out = append(out, d)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// HasDirective reports whether meta contains a directive with the given
|
||||
// namespace and key.
|
||||
func HasDirective(meta map[string]any, ns, key string) bool {
|
||||
for _, d := range GetDirectives(meta) {
|
||||
if d.Namespace == ns && d.Key == key {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// DirectiveSpec describes a documented directive in the catalog.
|
||||
type DirectiveSpec struct {
|
||||
// Singleton means only one directive with this namespace/key may appear at
|
||||
// a single location; a second one is a parse error.
|
||||
Singleton bool
|
||||
// Locations lists the location kinds the directive is valid at
|
||||
// ("database", "table", "column", "index").
|
||||
Locations []string
|
||||
}
|
||||
|
||||
// Location kinds a directive may attach to.
|
||||
const (
|
||||
DirectiveLocationDatabase = "database"
|
||||
DirectiveLocationTable = "table"
|
||||
DirectiveLocationColumn = "column"
|
||||
DirectiveLocationIndex = "index"
|
||||
)
|
||||
|
||||
// DirectiveCatalog is the set of documented directives per namespace. It is used
|
||||
// for strict-mode validation in readers and writers; unknown namespaces/keys are
|
||||
// still preserved losslessly when strict mode is off.
|
||||
var DirectiveCatalog = map[string]map[string]DirectiveSpec{
|
||||
"postgres": {
|
||||
"partition": {Singleton: true, Locations: []string{DirectiveLocationTable}},
|
||||
"tablespace": {Singleton: true, Locations: []string{DirectiveLocationTable, DirectiveLocationIndex}},
|
||||
"inherits": {Singleton: true, Locations: []string{DirectiveLocationTable}},
|
||||
"with": {Singleton: false, Locations: []string{DirectiveLocationTable, DirectiveLocationIndex}},
|
||||
"storage": {Singleton: true, Locations: []string{DirectiveLocationColumn}},
|
||||
"compression": {Singleton: true, Locations: []string{DirectiveLocationColumn}},
|
||||
"identity": {Singleton: true, Locations: []string{DirectiveLocationColumn}},
|
||||
},
|
||||
"sqlite": {
|
||||
"without": {Singleton: true, Locations: []string{DirectiveLocationTable}},
|
||||
"strict": {Singleton: true, Locations: []string{DirectiveLocationTable}},
|
||||
"collate": {Singleton: true, Locations: []string{DirectiveLocationColumn}},
|
||||
},
|
||||
}
|
||||
|
||||
// LookupDirectiveSpec returns the catalog spec for a namespace/key and whether
|
||||
// it is documented.
|
||||
func LookupDirectiveSpec(ns, key string) (DirectiveSpec, bool) {
|
||||
keys, ok := DirectiveCatalog[ns]
|
||||
if !ok {
|
||||
return DirectiveSpec{}, false
|
||||
}
|
||||
spec, ok := keys[key]
|
||||
return spec, ok
|
||||
}
|
||||
|
||||
// DirectiveLocationAllowed reports whether a documented directive may appear at
|
||||
// the given location. Unknown directives (not in the catalog) are allowed
|
||||
// everywhere so they can be preserved.
|
||||
func DirectiveLocationAllowed(ns, key, location string) bool {
|
||||
spec, ok := LookupDirectiveSpec(ns, key)
|
||||
if !ok {
|
||||
return true
|
||||
}
|
||||
for _, l := range spec.Locations {
|
||||
if l == location {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// FormatDirectiveLine renders a directive back to its DBML source form, e.g.
|
||||
// "@postgres: partition by RANGE (created_at)" or "@postgres(id): identity always".
|
||||
func FormatDirectiveLine(d Directive, target string) string {
|
||||
if target != "" {
|
||||
return fmt.Sprintf("@%s(%s): %s", d.Namespace, target, d.Args)
|
||||
}
|
||||
return fmt.Sprintf("@%s: %s", d.Namespace, d.Args)
|
||||
}
|
||||
@@ -0,0 +1,133 @@
|
||||
package models
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestDirectiveKey(t *testing.T) {
|
||||
cases := map[string]string{
|
||||
"partition by RANGE (created_at)": "partition",
|
||||
"WITHOUT ROWID": "without",
|
||||
" strict ": "strict",
|
||||
"": "",
|
||||
}
|
||||
for args, want := range cases {
|
||||
if got := DirectiveKey(args); got != want {
|
||||
t.Errorf("DirectiveKey(%q) = %q, want %q", args, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestAddDirectiveDerivesKey(t *testing.T) {
|
||||
meta := map[string]any{}
|
||||
AddDirective(meta, Directive{Namespace: "postgres", Args: "partition by RANGE (x)", Line: 2})
|
||||
AddDirective(meta, Directive{Namespace: "postgres", Key: "tablespace", Args: "tablespace fast", Line: 3})
|
||||
|
||||
got := GetDirectives(meta)
|
||||
if len(got) != 2 {
|
||||
t.Fatalf("got %d directives, want 2", len(got))
|
||||
}
|
||||
if got[0].Key != "partition" {
|
||||
t.Errorf("derived key = %q, want %q", got[0].Key, "partition")
|
||||
}
|
||||
if got[1].Key != "tablespace" {
|
||||
t.Errorf("explicit key = %q, want %q", got[1].Key, "tablespace")
|
||||
}
|
||||
}
|
||||
|
||||
func TestAddDirectiveNilMeta(t *testing.T) {
|
||||
// Must not panic.
|
||||
AddDirective(nil, Directive{Namespace: "postgres", Args: "strict"})
|
||||
}
|
||||
|
||||
func TestGetDirectivesOrdering(t *testing.T) {
|
||||
meta := map[string]any{}
|
||||
AddDirective(meta, Directive{Namespace: "sqlite", Args: "strict", Line: 9})
|
||||
AddDirective(meta, Directive{Namespace: "postgres", Args: "with (b)", Line: 5})
|
||||
AddDirective(meta, Directive{Namespace: "postgres", Args: "with (a)", Line: 5})
|
||||
AddDirective(meta, Directive{Namespace: "postgres", Args: "partition by x", Line: 2})
|
||||
|
||||
got := GetDirectives(meta)
|
||||
wantArgs := []string{"partition by x", "with (a)", "with (b)", "strict"}
|
||||
if len(got) != len(wantArgs) {
|
||||
t.Fatalf("got %d directives, want %d", len(got), len(wantArgs))
|
||||
}
|
||||
for i, w := range wantArgs {
|
||||
if got[i].Args != w {
|
||||
t.Errorf("directive[%d].Args = %q, want %q", i, got[i].Args, w)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetDirectivesTolerantDecodeAfterJSON(t *testing.T) {
|
||||
meta := map[string]any{}
|
||||
AddDirective(meta, Directive{Namespace: "postgres", Args: "partition by RANGE (created_at)", Line: 4})
|
||||
AddDirective(meta, Directive{Namespace: "sqlite", Args: "without rowid", Line: 6})
|
||||
|
||||
blob, err := json.Marshal(meta)
|
||||
if err != nil {
|
||||
t.Fatalf("marshal: %v", err)
|
||||
}
|
||||
var round map[string]any
|
||||
if err := json.Unmarshal(blob, &round); err != nil {
|
||||
t.Fatalf("unmarshal: %v", err)
|
||||
}
|
||||
|
||||
got := GetDirectives(round)
|
||||
if len(got) != 2 {
|
||||
t.Fatalf("got %d directives after JSON round-trip, want 2", len(got))
|
||||
}
|
||||
if got[0].Namespace != "postgres" || got[0].Key != "partition" || got[0].Line != 4 {
|
||||
t.Errorf("post-JSON directive[0] = %+v", got[0])
|
||||
}
|
||||
if got[0].Args != "partition by RANGE (created_at)" {
|
||||
t.Errorf("post-JSON args not verbatim: %q", got[0].Args)
|
||||
}
|
||||
if got[1].Namespace != "sqlite" || got[1].Key != "without" {
|
||||
t.Errorf("post-JSON directive[1] = %+v", got[1])
|
||||
}
|
||||
}
|
||||
|
||||
func TestDirectivesForNamespaceAndHasDirective(t *testing.T) {
|
||||
meta := map[string]any{}
|
||||
AddDirective(meta, Directive{Namespace: "postgres", Args: "partition by x", Line: 1})
|
||||
AddDirective(meta, Directive{Namespace: "sqlite", Args: "strict", Line: 2})
|
||||
|
||||
pg := DirectivesForNamespace(meta, "postgres")
|
||||
if len(pg) != 1 || pg[0].Key != "partition" {
|
||||
t.Errorf("DirectivesForNamespace(postgres) = %+v", pg)
|
||||
}
|
||||
if !HasDirective(meta, "sqlite", "strict") {
|
||||
t.Error("HasDirective(sqlite, strict) = false, want true")
|
||||
}
|
||||
if HasDirective(meta, "postgres", "tablespace") {
|
||||
t.Error("HasDirective(postgres, tablespace) = true, want false")
|
||||
}
|
||||
}
|
||||
|
||||
func TestDirectiveLocationAllowed(t *testing.T) {
|
||||
if !DirectiveLocationAllowed("postgres", "partition", DirectiveLocationTable) {
|
||||
t.Error("partition should be allowed at table level")
|
||||
}
|
||||
if DirectiveLocationAllowed("postgres", "partition", DirectiveLocationColumn) {
|
||||
t.Error("partition should not be allowed at column level")
|
||||
}
|
||||
// Unknown directives are allowed everywhere so they can be preserved.
|
||||
if !DirectiveLocationAllowed("postgres", "bogus", DirectiveLocationDatabase) {
|
||||
t.Error("unknown key should be allowed everywhere")
|
||||
}
|
||||
if !DirectiveLocationAllowed("madeup", "x", DirectiveLocationTable) {
|
||||
t.Error("unknown namespace should be allowed everywhere")
|
||||
}
|
||||
}
|
||||
|
||||
func TestFormatDirectiveLine(t *testing.T) {
|
||||
d := Directive{Namespace: "postgres", Key: "identity", Args: "identity always"}
|
||||
if got := FormatDirectiveLine(d, ""); got != "@postgres: identity always" {
|
||||
t.Errorf("FormatDirectiveLine no target = %q", got)
|
||||
}
|
||||
if got := FormatDirectiveLine(d, "id"); got != "@postgres(id): identity always" {
|
||||
t.Errorf("FormatDirectiveLine with target = %q", got)
|
||||
}
|
||||
}
|
||||
+64
-53
@@ -20,20 +20,22 @@ const (
|
||||
PostgresqlDatabaseType DatabaseType = "pgsql" // PostgreSQL database
|
||||
MSSQLDatabaseType DatabaseType = "mssql" // Microsoft SQL Server database
|
||||
SqlLiteDatabaseType DatabaseType = "sqlite" // SQLite database
|
||||
MySQLDatabaseType DatabaseType = "mysql" // MySQL/MariaDB database
|
||||
)
|
||||
|
||||
// Database represents the complete database schema
|
||||
type Database struct {
|
||||
Name string `json:"name" yaml:"name"`
|
||||
Description string `json:"description,omitempty" yaml:"description,omitempty" xml:"description,omitempty"`
|
||||
Schemas []*Schema `json:"schemas" yaml:"schemas" xml:"schemas"`
|
||||
Domains []*Domain `json:"domains,omitempty" yaml:"domains,omitempty" xml:"domains,omitempty"`
|
||||
Comment string `json:"comment,omitempty" yaml:"comment,omitempty" xml:"comment,omitempty"`
|
||||
DatabaseType DatabaseType `json:"database_type,omitempty" yaml:"database_type,omitempty" xml:"database_type,omitempty"`
|
||||
DatabaseVersion string `json:"database_version,omitempty" yaml:"database_version,omitempty" xml:"database_version,omitempty"`
|
||||
SourceFormat string `json:"source_format,omitempty" yaml:"source_format,omitempty" xml:"source_format,omitempty"` // Source Format of the database.
|
||||
UpdatedAt string `json:"updatedat,omitempty" yaml:"updatedat,omitempty" xml:"updatedat,omitempty"`
|
||||
GUID string `json:"guid" yaml:"guid" xml:"guid"`
|
||||
Name string `json:"name" yaml:"name"`
|
||||
Description string `json:"description,omitempty" yaml:"description,omitempty" xml:"description,omitempty"`
|
||||
Schemas []*Schema `json:"schemas" yaml:"schemas" xml:"schemas"`
|
||||
Domains []*Domain `json:"domains,omitempty" yaml:"domains,omitempty" xml:"domains,omitempty"`
|
||||
Comment string `json:"comment,omitempty" yaml:"comment,omitempty" xml:"comment,omitempty"`
|
||||
DatabaseType DatabaseType `json:"database_type,omitempty" yaml:"database_type,omitempty" xml:"database_type,omitempty"`
|
||||
DatabaseVersion string `json:"database_version,omitempty" yaml:"database_version,omitempty" xml:"database_version,omitempty"`
|
||||
SourceFormat string `json:"source_format,omitempty" yaml:"source_format,omitempty" xml:"source_format,omitempty"` // Source Format of the database.
|
||||
Metadata map[string]any `json:"metadata,omitempty" yaml:"metadata,omitempty" xml:"-"`
|
||||
UpdatedAt string `json:"updatedat,omitempty" yaml:"updatedat,omitempty" xml:"updatedat,omitempty"`
|
||||
GUID string `json:"guid" yaml:"guid" xml:"guid"`
|
||||
}
|
||||
|
||||
// SQLName returns the database name in lowercase for SQL compatibility.
|
||||
@@ -226,22 +228,27 @@ func (d *Sequence) SQLName() string {
|
||||
|
||||
// Column represents a table column
|
||||
type Column struct {
|
||||
Name string `json:"name" yaml:"name" xml:"name"`
|
||||
Description string `json:"description,omitempty" yaml:"description,omitempty" xml:"description,omitempty"`
|
||||
Table string `json:"table" yaml:"table" xml:"table"`
|
||||
Schema string `json:"schema" yaml:"schema" xml:"schema"`
|
||||
Type string `json:"type" yaml:"type" xml:"type"`
|
||||
Length int `json:"length,omitempty" yaml:"length,omitempty" xml:"length,omitempty"`
|
||||
Precision int `json:"precision,omitempty" yaml:"precision,omitempty" xml:"precision,omitempty"`
|
||||
Scale int `json:"scale,omitempty" yaml:"scale,omitempty" xml:"scale,omitempty"`
|
||||
NotNull bool `json:"not_null" yaml:"not_null" xml:"not_null"`
|
||||
Default any `json:"default,omitempty" yaml:"default,omitempty" xml:"default,omitempty"`
|
||||
AutoIncrement bool `json:"auto_increment" yaml:"auto_increment" xml:"auto_increment"`
|
||||
IsPrimaryKey bool `json:"is_primary_key" yaml:"is_primary_key" xml:"is_primary_key"`
|
||||
Comment string `json:"comment,omitempty" yaml:"comment,omitempty" xml:"comment,omitempty"`
|
||||
Collation string `json:"collation,omitempty" yaml:"collation,omitempty" xml:"collation,omitempty"`
|
||||
Sequence uint `json:"sequence,omitempty" yaml:"sequence,omitempty" xml:"sequence,omitempty"`
|
||||
GUID string `json:"guid" yaml:"guid" xml:"guid"`
|
||||
Name string `json:"name" yaml:"name" xml:"name"`
|
||||
Description string `json:"description,omitempty" yaml:"description,omitempty" xml:"description,omitempty"`
|
||||
Table string `json:"table" yaml:"table" xml:"table"`
|
||||
Schema string `json:"schema" yaml:"schema" xml:"schema"`
|
||||
Type string `json:"type" yaml:"type" xml:"type"`
|
||||
Length int `json:"length,omitempty" yaml:"length,omitempty" xml:"length,omitempty"`
|
||||
Precision int `json:"precision,omitempty" yaml:"precision,omitempty" xml:"precision,omitempty"`
|
||||
Scale int `json:"scale,omitempty" yaml:"scale,omitempty" xml:"scale,omitempty"`
|
||||
NotNull bool `json:"not_null" yaml:"not_null" xml:"not_null"`
|
||||
Default any `json:"default,omitempty" yaml:"default,omitempty" xml:"default,omitempty"`
|
||||
AutoIncrement bool `json:"auto_increment" yaml:"auto_increment" xml:"auto_increment"`
|
||||
IsPrimaryKey bool `json:"is_primary_key" yaml:"is_primary_key" xml:"is_primary_key"`
|
||||
Comment string `json:"comment,omitempty" yaml:"comment,omitempty" xml:"comment,omitempty"`
|
||||
Collation string `json:"collation,omitempty" yaml:"collation,omitempty" xml:"collation,omitempty"`
|
||||
Metadata map[string]any `json:"metadata,omitempty" yaml:"metadata,omitempty" xml:"-"`
|
||||
Sequence uint `json:"sequence,omitempty" yaml:"sequence,omitempty" xml:"sequence,omitempty"`
|
||||
GUID string `json:"guid" yaml:"guid" xml:"guid"`
|
||||
Generated bool `json:"generated,omitempty" yaml:"generated,omitempty" xml:"generated,omitempty"`
|
||||
GenerationExpression string `json:"generation_expression,omitempty" yaml:"generation_expression,omitempty" xml:"generation_expression,omitempty"`
|
||||
Identity bool `json:"identity,omitempty" yaml:"identity,omitempty" xml:"identity,omitempty"`
|
||||
IdentityGeneration string `json:"identity_generation,omitempty" yaml:"identity_generation,omitempty" xml:"identity_generation,omitempty"` // "ALWAYS" or "BY DEFAULT"
|
||||
}
|
||||
|
||||
// SQLName returns the column name in lowercase for SQL compatibility.
|
||||
@@ -252,19 +259,20 @@ func (d *Column) SQLName() string {
|
||||
// Index represents a database index for optimizing query performance.
|
||||
// Indexes can be unique, partial, or include additional columns.
|
||||
type Index struct {
|
||||
Name string `json:"name" yaml:"name" xml:"name"`
|
||||
Description string `json:"description,omitempty" yaml:"description,omitempty" xml:"description,omitempty"`
|
||||
Table string `json:"table,omitempty" yaml:"table,omitempty" xml:"table,omitempty"`
|
||||
Schema string `json:"schema,omitempty" yaml:"schema,omitempty" xml:"schema,omitempty"`
|
||||
Columns []string `json:"columns" yaml:"columns" xml:"columns"`
|
||||
Unique bool `json:"unique" yaml:"unique" xml:"unique"`
|
||||
Type string `json:"type" yaml:"type" xml:"type"` // btree, hash, gin, gist, etc.
|
||||
Where string `json:"where,omitempty" yaml:"where,omitempty" xml:"where,omitempty"` // partial index condition
|
||||
Concurrent bool `json:"concurrent,omitempty" yaml:"concurrent,omitempty" xml:"concurrent,omitempty"`
|
||||
Include []string `json:"include,omitempty" yaml:"include,omitempty" xml:"include,omitempty"` // INCLUDE columns
|
||||
Comment string `json:"comment,omitempty" yaml:"comment,omitempty" xml:"comment,omitempty"`
|
||||
Sequence uint `json:"sequence,omitempty" yaml:"sequence,omitempty" xml:"sequence,omitempty"`
|
||||
GUID string `json:"guid" yaml:"guid" xml:"guid"`
|
||||
Name string `json:"name" yaml:"name" xml:"name"`
|
||||
Description string `json:"description,omitempty" yaml:"description,omitempty" xml:"description,omitempty"`
|
||||
Table string `json:"table,omitempty" yaml:"table,omitempty" xml:"table,omitempty"`
|
||||
Schema string `json:"schema,omitempty" yaml:"schema,omitempty" xml:"schema,omitempty"`
|
||||
Columns []string `json:"columns" yaml:"columns" xml:"columns"`
|
||||
Unique bool `json:"unique" yaml:"unique" xml:"unique"`
|
||||
Type string `json:"type" yaml:"type" xml:"type"` // btree, hash, gin, gist, etc.
|
||||
Where string `json:"where,omitempty" yaml:"where,omitempty" xml:"where,omitempty"` // partial index condition
|
||||
Concurrent bool `json:"concurrent,omitempty" yaml:"concurrent,omitempty" xml:"concurrent,omitempty"`
|
||||
Include []string `json:"include,omitempty" yaml:"include,omitempty" xml:"include,omitempty"` // INCLUDE columns
|
||||
Comment string `json:"comment,omitempty" yaml:"comment,omitempty" xml:"comment,omitempty"`
|
||||
Metadata map[string]any `json:"metadata,omitempty" yaml:"metadata,omitempty" xml:"-"`
|
||||
Sequence uint `json:"sequence,omitempty" yaml:"sequence,omitempty" xml:"sequence,omitempty"`
|
||||
GUID string `json:"guid" yaml:"guid" xml:"guid"`
|
||||
}
|
||||
|
||||
// SQLName returns the index name in lowercase for SQL compatibility.
|
||||
@@ -393,10 +401,11 @@ func (d *Script) SQLName() string {
|
||||
// InitDatabase initializes a new Database with empty slices
|
||||
func InitDatabase(name string) *Database {
|
||||
return &Database{
|
||||
Name: name,
|
||||
Schemas: make([]*Schema, 0),
|
||||
Domains: make([]*Domain, 0),
|
||||
GUID: uuid.New().String(),
|
||||
Name: name,
|
||||
Schemas: make([]*Schema, 0),
|
||||
Domains: make([]*Domain, 0),
|
||||
Metadata: make(map[string]any),
|
||||
GUID: uuid.New().String(),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -431,22 +440,24 @@ func InitTable(name, schema string) *Table {
|
||||
// InitColumn initializes a new Column
|
||||
func InitColumn(name, table, schema string) *Column {
|
||||
return &Column{
|
||||
Name: name,
|
||||
Table: table,
|
||||
Schema: schema,
|
||||
GUID: uuid.New().String(),
|
||||
Name: name,
|
||||
Table: table,
|
||||
Schema: schema,
|
||||
Metadata: make(map[string]any),
|
||||
GUID: uuid.New().String(),
|
||||
}
|
||||
}
|
||||
|
||||
// InitIndex initializes a new Index with empty slices
|
||||
func InitIndex(name, table, schema string) *Index {
|
||||
return &Index{
|
||||
Name: name,
|
||||
Table: table,
|
||||
Schema: schema,
|
||||
Columns: make([]string, 0),
|
||||
Include: make([]string, 0),
|
||||
GUID: uuid.New().String(),
|
||||
Name: name,
|
||||
Table: table,
|
||||
Schema: schema,
|
||||
Columns: make([]string, 0),
|
||||
Include: make([]string, 0),
|
||||
Metadata: make(map[string]any),
|
||||
GUID: uuid.New().String(),
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -247,7 +247,8 @@ var extensionFunctionPrefixes = buildExtensionIndex(func(ext Extension) []string
|
||||
|
||||
func buildExtensionIndex(keys func(Extension) []string) map[string]string {
|
||||
index := make(map[string]string)
|
||||
for _, ext := range postgresExtensions {
|
||||
for name := range postgresExtensions {
|
||||
ext := postgresExtensions[name]
|
||||
for _, key := range keys(ext) {
|
||||
// Deterministic on collision: the alphabetically first extension wins.
|
||||
if existing, ok := index[key]; ok && existing < ext.Name {
|
||||
|
||||
@@ -64,6 +64,8 @@ The reader recognizes the following Bun struct tags:
|
||||
- `autoincrement` - Auto-increment column
|
||||
- `default` - Default value
|
||||
- `unique` - Unique constraint
|
||||
- `generated` - Generated column marker (`Generated`; expression is not in the tag)
|
||||
- `identity` - Identity column marker (`Identity`, `ALWAYS`)
|
||||
- `rel` - Relationship definition
|
||||
|
||||
## Example Bun Model
|
||||
|
||||
@@ -0,0 +1,48 @@
|
||||
package bun
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"git.warky.dev/wdevs/relspecgo/pkg/readers"
|
||||
)
|
||||
|
||||
func TestReader_GeneratedAndIdentityMarkers(t *testing.T) {
|
||||
src := `package models
|
||||
|
||||
import "github.com/uptrace/bun"
|
||||
|
||||
type Person struct {
|
||||
bun.BaseModel ` + "`bun:\"table:people,alias:p\"`" + `
|
||||
|
||||
ID int64 ` + "`bun:\"id,pk,type:bigint,autoincrement,identity\"`" + `
|
||||
Seq int64 ` + "`bun:\"seq,type:bigint,scanonly,identity,notnull\"`" + `
|
||||
FullName string ` + "`bun:\"full_name,type:text,scanonly,generated,nullzero\"`" + `
|
||||
First string ` + "`bun:\"first,type:text,nullzero\"`" + `
|
||||
}
|
||||
`
|
||||
path := filepath.Join(t.TempDir(), "person.go")
|
||||
if err := os.WriteFile(path, []byte(src), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
db, err := NewReader(&readers.ReaderOptions{FilePath: path}).ReadDatabase()
|
||||
if err != nil {
|
||||
t.Fatalf("ReadDatabase() error = %v", err)
|
||||
}
|
||||
cols := db.Schemas[0].Tables[0].Columns
|
||||
|
||||
if !cols["id"].Identity || cols["id"].IdentityGeneration != "ALWAYS" {
|
||||
t.Errorf("id: identity = %v/%q, want true/ALWAYS", cols["id"].Identity, cols["id"].IdentityGeneration)
|
||||
}
|
||||
if !cols["seq"].Identity || cols["seq"].IdentityGeneration != "ALWAYS" {
|
||||
t.Errorf("seq: identity = %v/%q, want true/ALWAYS", cols["seq"].Identity, cols["seq"].IdentityGeneration)
|
||||
}
|
||||
if !cols["full_name"].Generated {
|
||||
t.Error("full_name should be generated")
|
||||
}
|
||||
if cols["first"].Generated || cols["first"].Identity {
|
||||
t.Error("first must be an ordinary column")
|
||||
}
|
||||
}
|
||||
@@ -245,7 +245,7 @@ func (r *Reader) getReceiverType(expr ast.Expr) string {
|
||||
}
|
||||
|
||||
// parseTableNameMethod parses a TableName() method and extracts the table and schema name
|
||||
func (r *Reader) parseTableNameMethod(funcDecl *ast.FuncDecl) (tableName string, schemaName string) {
|
||||
func (r *Reader) parseTableNameMethod(funcDecl *ast.FuncDecl) (tableName, schemaName string) {
|
||||
if funcDecl.Body == nil {
|
||||
return "", ""
|
||||
}
|
||||
@@ -578,7 +578,7 @@ func (r *Reader) parseIndexesFromTag(table *models.Table, column *models.Column,
|
||||
}
|
||||
|
||||
// extractTableNameFromTag extracts table and schema from bun tag
|
||||
func (r *Reader) extractTableNameFromTag(tag string) (tableName string, schemaName string) {
|
||||
func (r *Reader) extractTableNameFromTag(tag string) (tableName, schemaName string) {
|
||||
// Extract bun tag value
|
||||
re := regexp.MustCompile(`bun:"table:([^"]+)"`)
|
||||
matches := re.FindStringSubmatch(tag)
|
||||
@@ -659,6 +659,13 @@ func (r *Reader) parseColumn(fieldName string, fieldType ast.Expr, tag string, s
|
||||
hasExplicitNullableMarker = true
|
||||
case "autoincrement":
|
||||
column.AutoIncrement = true
|
||||
case "generated":
|
||||
// GENERATED ... STORED marker written by the Bun writer; the expression is not in the tag
|
||||
column.Generated = true
|
||||
case "identity":
|
||||
// GENERATED ALWAYS AS IDENTITY marker written by the Bun writer
|
||||
column.Identity = true
|
||||
column.IdentityGeneration = "ALWAYS"
|
||||
case "default":
|
||||
// Default value from Bun tag (e.g., default:gen_random_uuid())
|
||||
column.Default = value
|
||||
@@ -712,12 +719,12 @@ func (r *Reader) parseTypeWithLength(typeStr string) (baseType string, length in
|
||||
if pgsql.SupportsLength(rawBaseType) {
|
||||
if _, err := fmt.Sscanf(matches[2], "%d", &length); err == nil {
|
||||
baseType = pgsql.CanonicalizeBaseType(rawBaseType)
|
||||
return
|
||||
return baseType, length
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return
|
||||
return baseType, length
|
||||
}
|
||||
|
||||
// goTypeToSQL maps Go types to SQL types
|
||||
|
||||
@@ -90,12 +90,74 @@ Ref: posts.user_id > users.id [delete: cascade]
|
||||
- Default values (`default`)
|
||||
- Inline references (`ref`)
|
||||
- Standalone `Ref` blocks
|
||||
- Commented cross-file refs (`// Ref:` — see below)
|
||||
- Indexes and composite indexes
|
||||
- Table notes and column notes
|
||||
- Enums
|
||||
- Dialect directives (`@postgres:` / `@sqlite:` — see below)
|
||||
|
||||
## Commented cross-file refs
|
||||
|
||||
`// Ref:` / `// ref:` lines (ignored by dbdiagram) become FKs + relationships once both ends are loaded.
|
||||
|
||||
| Rule | Behaviour |
|
||||
|---|---|
|
||||
| When | Single file / directory: end of read. `--from-list`, `merge`, jobs: after all inputs are combined |
|
||||
| Match | `schema.table.column` on both sides, case-insensitive |
|
||||
| Operators | `>`, `<`, `-` (parsed like `Ref:`) |
|
||||
| Duplicate of an FK on the same columns | Skipped silently |
|
||||
| Target not loaded | Kept pending; skipped with a warning on the final pass |
|
||||
| Column type mismatch | Warning, FK still created (`serial`≈`integer`, `bigserial`≈`bigint`) |
|
||||
| Pending state | `Database.Metadata["dbml.commented_refs"]` (`[]string`) |
|
||||
|
||||
API: `dbml.ResolveCommentedRefs(db, final bool) []string` returns warnings.
|
||||
|
||||
## Dialect directives
|
||||
|
||||
Lines of the form `@<namespace>[(<column>)]: <args>` embed database-specific
|
||||
features that plain DBML cannot express (partitioning, `WITHOUT ROWID`,
|
||||
tablespaces, index storage parameters, …). They are stored losslessly on the
|
||||
relevant object's `Metadata` and round-trip unchanged through the DBML writer;
|
||||
the PostgreSQL and SQLite writers translate the ones they understand to SQL.
|
||||
|
||||
```dbml
|
||||
@postgres: search_path myapp
|
||||
|
||||
Table myapp.events {
|
||||
id bigint [pk]
|
||||
created_at timestamp [not null]
|
||||
@postgres(id): identity always
|
||||
@postgres: partition by RANGE (created_at)
|
||||
@sqlite: without rowid
|
||||
|
||||
indexes {
|
||||
(created_at) [name: 'idx_events_created']
|
||||
@postgres: with (fillfactor=90)
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
| Position | Attaches to |
|
||||
|----------|-------------|
|
||||
| Before the first `Table {` | database |
|
||||
| Table body, no `(target)` | that table |
|
||||
| Table body, `(col)` target | column `col` (error if unknown) |
|
||||
| Inside `indexes { }` | the most recently listed index entry |
|
||||
|
||||
`args` is preserved verbatim; the **key** (lowercased first token) drives
|
||||
duplicate detection. Repeated directives are kept in order; catalog "singleton"
|
||||
keys error on a second occurrence at the same location. All errors are
|
||||
line-numbered.
|
||||
|
||||
`ReaderOptions.StrictDirectives` (CLI `--strict-directives`) turns an unknown
|
||||
namespace or key into an error instead of preserving it silently.
|
||||
|
||||
See [`docs/DBML_DIRECTIVES.md`](../../../docs/DBML_DIRECTIVES.md) for the full
|
||||
grammar and the supported-directive matrix.
|
||||
|
||||
## Notes
|
||||
|
||||
- Column notes `GENERATED ALWAYS AS (expr) STORED` and `GENERATED ALWAYS|BY DEFAULT AS IDENTITY` set `Generated`/`GenerationExpression` and `Identity`/`IdentityGeneration`; they are not kept as comments
|
||||
- DBML is designed for database documentation and diagramming
|
||||
- Schema name defaults to `public`
|
||||
- Relationship cardinality is preserved
|
||||
|
||||
@@ -0,0 +1,218 @@
|
||||
package dbml
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"git.warky.dev/wdevs/relspecgo/pkg/models"
|
||||
)
|
||||
|
||||
// CommentedRefsMetadataKey is the Database.Metadata key holding commented
|
||||
// `// Ref:` lines not yet resolved against the model ([]string).
|
||||
const CommentedRefsMetadataKey = "dbml.commented_refs"
|
||||
|
||||
// commentedRefRegex matches `// Ref: ...` and `// ref: ...`.
|
||||
var commentedRefRegex = regexp.MustCompile(`^//\s*[Rr]ef\s*:\s*(.+)$`)
|
||||
|
||||
// commentedRef returns the ref body of a trimmed `// Ref:` comment line.
|
||||
func commentedRef(line string) (string, bool) {
|
||||
m := commentedRefRegex.FindStringSubmatch(line)
|
||||
if m == nil {
|
||||
return "", false
|
||||
}
|
||||
ref := strings.TrimSpace(m[1])
|
||||
return ref, ref != ""
|
||||
}
|
||||
|
||||
// PendingCommentedRefs returns the commented refs not yet resolved.
|
||||
func PendingCommentedRefs(db *models.Database) []string {
|
||||
if db == nil || db.Metadata == nil {
|
||||
return nil
|
||||
}
|
||||
refs, _ := db.Metadata[CommentedRefsMetadataKey].([]string)
|
||||
return refs
|
||||
}
|
||||
|
||||
func setPendingCommentedRefs(db *models.Database, refs []string) {
|
||||
if len(refs) == 0 {
|
||||
if db.Metadata != nil {
|
||||
delete(db.Metadata, CommentedRefsMetadataKey)
|
||||
}
|
||||
return
|
||||
}
|
||||
if db.Metadata == nil {
|
||||
db.Metadata = make(map[string]any)
|
||||
}
|
||||
db.Metadata[CommentedRefsMetadataKey] = refs
|
||||
}
|
||||
|
||||
// addPendingCommentedRef queues a ref, skipping exact repeats.
|
||||
func addPendingCommentedRef(db *models.Database, ref string) {
|
||||
refs := PendingCommentedRefs(db)
|
||||
for _, existing := range refs {
|
||||
if existing == ref {
|
||||
return
|
||||
}
|
||||
}
|
||||
setPendingCommentedRefs(db, append(refs, ref))
|
||||
}
|
||||
|
||||
// ResolveCommentedRefs turns pending commented refs into foreign keys and
|
||||
// relationships when both ends (schema.table.column) exist in db. A ref that
|
||||
// matches an existing FK on the same columns is dropped as a duplicate.
|
||||
// Unmatched refs stay pending unless final is set, in which case they are
|
||||
// dropped with a warning. Returns human-readable warnings.
|
||||
func ResolveCommentedRefs(db *models.Database, final bool) []string {
|
||||
refs := PendingCommentedRefs(db)
|
||||
if len(refs) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
var warnings []string
|
||||
var pending []string
|
||||
parser := &Reader{}
|
||||
|
||||
for _, ref := range refs {
|
||||
fk := parser.parseRef(ref)
|
||||
if fk == nil || len(fk.Columns) == 0 || len(fk.Columns) != len(fk.ReferencedColumns) {
|
||||
warnings = append(warnings, fmt.Sprintf("skipping commented ref %q: cannot parse", ref))
|
||||
continue
|
||||
}
|
||||
|
||||
srcTable, srcCols, srcMissing := lookupColumns(db, fk.Schema, fk.Table, fk.Columns)
|
||||
dstTable, dstCols, dstMissing := lookupColumns(db, fk.ReferencedSchema, fk.ReferencedTable, fk.ReferencedColumns)
|
||||
if srcMissing != "" || dstMissing != "" {
|
||||
if final {
|
||||
missing := srcMissing
|
||||
if missing == "" {
|
||||
missing = dstMissing
|
||||
}
|
||||
warnings = append(warnings, fmt.Sprintf("skipping commented ref %q: %s not found", ref, missing))
|
||||
} else {
|
||||
pending = append(pending, ref)
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
fk.Schema, fk.Table, fk.Columns = srcTable.Schema, srcTable.Name, columnNames(srcCols)
|
||||
fk.ReferencedSchema, fk.ReferencedTable, fk.ReferencedColumns = dstTable.Schema, dstTable.Name, columnNames(dstCols)
|
||||
|
||||
if hasFKOnColumns(srcTable, fk.Columns) {
|
||||
continue // already declared by an uncommented Ref or inline ref
|
||||
}
|
||||
if _, taken := srcTable.Constraints[fk.Name]; taken {
|
||||
warnings = append(warnings, fmt.Sprintf("skipping commented ref %q: constraint %s already exists", ref, fk.Name))
|
||||
continue
|
||||
}
|
||||
|
||||
for i := range srcCols {
|
||||
if !compatibleFKTypes(srcCols[i].Type, dstCols[i].Type) {
|
||||
warnings = append(warnings, fmt.Sprintf("commented ref %q: type mismatch %s.%s.%s (%s) -> %s.%s.%s (%s)",
|
||||
ref, srcTable.Schema, srcTable.Name, srcCols[i].Name, srcCols[i].Type,
|
||||
dstTable.Schema, dstTable.Name, dstCols[i].Name, dstCols[i].Type))
|
||||
}
|
||||
}
|
||||
|
||||
if srcTable.Constraints == nil {
|
||||
srcTable.Constraints = make(map[string]*models.Constraint)
|
||||
}
|
||||
srcTable.Constraints[fk.Name] = fk
|
||||
addFKRelationship(srcTable, fk)
|
||||
}
|
||||
|
||||
setPendingCommentedRefs(db, pending)
|
||||
return warnings
|
||||
}
|
||||
|
||||
// lookupColumns finds a table and its columns, case-insensitively. missing
|
||||
// names the first object not found, or is empty.
|
||||
func lookupColumns(db *models.Database, schemaName, tableName string, cols []string) (*models.Table, []*models.Column, string) {
|
||||
qualified := schemaName + "." + tableName
|
||||
var table *models.Table
|
||||
for _, schema := range db.Schemas {
|
||||
if !strings.EqualFold(schema.Name, schemaName) {
|
||||
continue
|
||||
}
|
||||
for _, t := range schema.Tables {
|
||||
if strings.EqualFold(t.Name, tableName) {
|
||||
table = t
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
if table == nil {
|
||||
return nil, nil, "table " + qualified
|
||||
}
|
||||
|
||||
found := make([]*models.Column, 0, len(cols))
|
||||
for _, name := range cols {
|
||||
col := table.Columns[name]
|
||||
if col == nil {
|
||||
for _, c := range table.Columns {
|
||||
if strings.EqualFold(c.Name, name) {
|
||||
col = c
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
if col == nil {
|
||||
return nil, nil, "column " + qualified + "." + name
|
||||
}
|
||||
found = append(found, col)
|
||||
}
|
||||
return table, found, ""
|
||||
}
|
||||
|
||||
func columnNames(cols []*models.Column) []string {
|
||||
names := make([]string, len(cols))
|
||||
for i, c := range cols {
|
||||
names[i] = c.Name
|
||||
}
|
||||
return names
|
||||
}
|
||||
|
||||
// hasFKOnColumns reports whether table already has a foreign key over cols.
|
||||
func hasFKOnColumns(table *models.Table, cols []string) bool {
|
||||
for _, c := range table.Constraints {
|
||||
if c.Type != models.ForeignKeyConstraint || len(c.Columns) != len(cols) {
|
||||
continue
|
||||
}
|
||||
same := true
|
||||
for i := range cols {
|
||||
if !strings.EqualFold(c.Columns[i], cols[i]) {
|
||||
same = false
|
||||
break
|
||||
}
|
||||
}
|
||||
if same {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// fkTypeAliases maps serial and alias spellings to their storage type.
|
||||
var fkTypeAliases = map[string]string{
|
||||
"smallserial": "smallint", "serial2": "smallint", "int2": "smallint",
|
||||
"serial": "integer", "serial4": "integer", "int": "integer", "int4": "integer",
|
||||
"bigserial": "bigint", "serial8": "bigint", "int8": "bigint",
|
||||
}
|
||||
|
||||
// compatibleFKTypes compares column types ignoring case, length and serial
|
||||
// vs. integer spelling. Unknown (empty) types are treated as compatible.
|
||||
func compatibleFKTypes(a, b string) bool {
|
||||
na, nb := normalizeFKType(a), normalizeFKType(b)
|
||||
return na == "" || nb == "" || na == nb
|
||||
}
|
||||
|
||||
func normalizeFKType(t string) string {
|
||||
t = strings.ToLower(strings.TrimSpace(t))
|
||||
if i := strings.Index(t, "("); i >= 0 {
|
||||
t = strings.TrimSpace(t[:i])
|
||||
}
|
||||
if alias, ok := fkTypeAliases[t]; ok {
|
||||
return alias
|
||||
}
|
||||
return t
|
||||
}
|
||||
@@ -0,0 +1,260 @@
|
||||
package dbml
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.warky.dev/wdevs/relspecgo/pkg/models"
|
||||
"git.warky.dev/wdevs/relspecgo/pkg/readers"
|
||||
)
|
||||
|
||||
func readDBMLString(t *testing.T, content string) *models.Database {
|
||||
t.Helper()
|
||||
path := filepath.Join(t.TempDir(), "in.dbml")
|
||||
if err := os.WriteFile(path, []byte(content), 0o644); err != nil {
|
||||
t.Fatalf("write fixture: %v", err)
|
||||
}
|
||||
db, err := NewReader(&readers.ReaderOptions{FilePath: path}).ReadDatabase()
|
||||
if err != nil {
|
||||
t.Fatalf("ReadDatabase() error = %v", err)
|
||||
}
|
||||
return db
|
||||
}
|
||||
|
||||
func findTable(db *models.Database, schema, table string) *models.Table {
|
||||
for _, s := range db.Schemas {
|
||||
if s.Name != schema {
|
||||
continue
|
||||
}
|
||||
for _, t := range s.Tables {
|
||||
if t.Name == table {
|
||||
return t
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func fkOn(table *models.Table, col string) *models.Constraint {
|
||||
for _, c := range table.Constraints {
|
||||
if c.Type == models.ForeignKeyConstraint && len(c.Columns) == 1 && c.Columns[0] == col {
|
||||
return c
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func relFor(table *models.Table, fkName string) *models.Relationship {
|
||||
for _, r := range table.Relationships {
|
||||
if r.ForeignKey == fkName {
|
||||
return r
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func TestCommentedRef(t *testing.T) {
|
||||
tests := []struct {
|
||||
line string
|
||||
want string
|
||||
ok bool
|
||||
}{
|
||||
{`// Ref: a.b.c > d.e.f`, `a.b.c > d.e.f`, true},
|
||||
{`// ref: a.b.c - d.e.f`, `a.b.c - d.e.f`, true},
|
||||
{`//Ref:a.b.c > d.e.f`, `a.b.c > d.e.f`, true},
|
||||
{`// Reference notes`, "", false},
|
||||
{`// see Ref: a.b.c > d.e.f`, "", false},
|
||||
{`// Ref:`, "", false},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
got, ok := commentedRef(tt.line)
|
||||
if ok != tt.ok || got != tt.want {
|
||||
t.Errorf("commentedRef(%q) = (%q, %v), want (%q, %v)", tt.line, got, ok, tt.want, tt.ok)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A commented ref whose tables are in the same file resolves on read.
|
||||
func TestReader_CommentedRefSameFile(t *testing.T) {
|
||||
db := readDBMLString(t, `Table "org"."department" {
|
||||
"id_department" bigserial [pk]
|
||||
}
|
||||
Table "entity"."employee" {
|
||||
"id_employee" bigserial [pk]
|
||||
"rid_department" bigint
|
||||
}
|
||||
// Ref: "entity"."employee"."rid_department" > "org"."department"."id_department" [delete: restrict, update: restrict]
|
||||
`)
|
||||
emp := findTable(db, "entity", "employee")
|
||||
fk := fkOn(emp, "rid_department")
|
||||
if fk == nil {
|
||||
t.Fatal("expected FK on rid_department")
|
||||
}
|
||||
if fk.ReferencedSchema != "org" || fk.ReferencedTable != "department" || fk.ReferencedColumns[0] != "id_department" {
|
||||
t.Errorf("FK target = %s.%s.%v", fk.ReferencedSchema, fk.ReferencedTable, fk.ReferencedColumns)
|
||||
}
|
||||
if fk.OnDelete != "restrict" || fk.OnUpdate != "restrict" {
|
||||
t.Errorf("FK actions = %q/%q, want restrict/restrict", fk.OnDelete, fk.OnUpdate)
|
||||
}
|
||||
if relFor(emp, fk.Name) == nil {
|
||||
t.Error("expected relationship for FK")
|
||||
}
|
||||
if refs := PendingCommentedRefs(db); len(refs) != 0 {
|
||||
t.Errorf("pending = %v, want none", refs)
|
||||
}
|
||||
}
|
||||
|
||||
// A cross-file commented ref stays pending after a single-file read.
|
||||
func TestReader_CommentedRefCrossFilePending(t *testing.T) {
|
||||
db := readDBMLString(t, `Table "entity"."employee" {
|
||||
"id_employee" bigserial [pk]
|
||||
"rid_department" bigint
|
||||
}
|
||||
// Ref: "entity"."employee"."rid_department" > "org"."department"."id_department"
|
||||
`)
|
||||
if fk := fkOn(findTable(db, "entity", "employee"), "rid_department"); fk != nil {
|
||||
t.Fatal("FK must not resolve without the target table")
|
||||
}
|
||||
if refs := PendingCommentedRefs(db); len(refs) != 1 {
|
||||
t.Fatalf("pending = %v, want 1", refs)
|
||||
}
|
||||
}
|
||||
|
||||
// Directory reads resolve commented refs after all files are merged.
|
||||
func TestReader_CommentedRefDirectory(t *testing.T) {
|
||||
db, err := NewReader(&readers.ReaderOptions{
|
||||
FilePath: filepath.Join("..", "..", "..", "tests", "assets", "dbml", "multifile"),
|
||||
}).ReadDatabase()
|
||||
if err != nil {
|
||||
t.Fatalf("ReadDatabase() error = %v", err)
|
||||
}
|
||||
fk := fkOn(findTable(db, "public", "posts"), "user_id")
|
||||
if fk == nil {
|
||||
t.Fatal("expected FK posts.user_id from 9_refs.dbml commented ref")
|
||||
}
|
||||
if fk.ReferencedTable != "users" || fk.OnDelete != "CASCADE" {
|
||||
t.Errorf("FK = %s ondelete %s, want users ondelete CASCADE", fk.ReferencedTable, fk.OnDelete)
|
||||
}
|
||||
}
|
||||
|
||||
func crossFileDB(t *testing.T, refs ...string) *models.Database {
|
||||
t.Helper()
|
||||
db := readDBMLString(t, `Table "org"."department" {
|
||||
"id_department" bigserial [pk]
|
||||
}
|
||||
Table "entity"."employee" {
|
||||
"id_employee" bigserial [pk]
|
||||
"rid_department" bigint
|
||||
"rid_manager" integer
|
||||
"rid_team" bigint
|
||||
}
|
||||
`)
|
||||
setPendingCommentedRefs(db, refs)
|
||||
return db
|
||||
}
|
||||
|
||||
func TestResolveCommentedRefs(t *testing.T) {
|
||||
t.Run("lowercase ref and one-to-one", func(t *testing.T) {
|
||||
db := crossFileDB(t,
|
||||
`"entity"."employee"."rid_department" > "org"."department"."id_department"`,
|
||||
`entity.employee.rid_team - org.department.id_department`,
|
||||
)
|
||||
if w := ResolveCommentedRefs(db, true); len(w) != 0 {
|
||||
t.Errorf("warnings = %v, want none", w)
|
||||
}
|
||||
emp := findTable(db, "entity", "employee")
|
||||
for _, col := range []string{"rid_department", "rid_team"} {
|
||||
fk := fkOn(emp, col)
|
||||
if fk == nil {
|
||||
t.Fatalf("expected FK on %s", col)
|
||||
}
|
||||
if relFor(emp, fk.Name) == nil {
|
||||
t.Errorf("expected relationship for %s", fk.Name)
|
||||
}
|
||||
}
|
||||
if len(emp.Relationships) != 2 {
|
||||
t.Errorf("relationships = %d, want 2 (same target must not overwrite)", len(emp.Relationships))
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("type mismatch warns but resolves", func(t *testing.T) {
|
||||
db := crossFileDB(t, `entity.employee.rid_manager > org.department.id_department`)
|
||||
w := ResolveCommentedRefs(db, true)
|
||||
if len(w) != 1 || !strings.Contains(w[0], "type mismatch") {
|
||||
t.Errorf("warnings = %v, want one type mismatch", w)
|
||||
}
|
||||
if fkOn(findTable(db, "entity", "employee"), "rid_manager") == nil {
|
||||
t.Error("expected FK on rid_manager")
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("missing target stays pending until final", func(t *testing.T) {
|
||||
db := crossFileDB(t, `entity.employee.rid_team > hr.team.id_team`)
|
||||
if w := ResolveCommentedRefs(db, false); len(w) != 0 {
|
||||
t.Errorf("non-final warnings = %v, want none", w)
|
||||
}
|
||||
if len(PendingCommentedRefs(db)) != 1 {
|
||||
t.Fatal("ref should stay pending")
|
||||
}
|
||||
w := ResolveCommentedRefs(db, true)
|
||||
if len(w) != 1 || !strings.Contains(w[0], "table hr.team not found") {
|
||||
t.Errorf("final warnings = %v, want missing table", w)
|
||||
}
|
||||
if len(PendingCommentedRefs(db)) != 0 {
|
||||
t.Error("final pass must clear pending refs")
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("missing column", func(t *testing.T) {
|
||||
db := crossFileDB(t, `entity.employee.rid_nope > org.department.id_department`)
|
||||
w := ResolveCommentedRefs(db, true)
|
||||
if len(w) != 1 || !strings.Contains(w[0], "column entity.employee.rid_nope not found") {
|
||||
t.Errorf("warnings = %v, want missing column", w)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("deduplicates against uncommented ref", func(t *testing.T) {
|
||||
db := readDBMLString(t, `Table "org"."department" {
|
||||
"id_department" bigserial [pk]
|
||||
}
|
||||
Table "entity"."employee" {
|
||||
"id_employee" bigserial [pk]
|
||||
"rid_department" bigint
|
||||
}
|
||||
Ref: "entity"."employee"."rid_department" > "org"."department"."id_department"
|
||||
// Ref: "entity"."employee"."rid_department" > "org"."department"."id_department"
|
||||
`)
|
||||
emp := findTable(db, "entity", "employee")
|
||||
count := 0
|
||||
for _, c := range emp.Constraints {
|
||||
if c.Type == models.ForeignKeyConstraint {
|
||||
count++
|
||||
}
|
||||
}
|
||||
if count != 1 || len(emp.Relationships) != 1 {
|
||||
t.Errorf("FKs = %d, relationships = %d, want 1 and 1", count, len(emp.Relationships))
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestCompatibleFKTypes(t *testing.T) {
|
||||
tests := []struct {
|
||||
a, b string
|
||||
want bool
|
||||
}{
|
||||
{"bigint", "bigserial", true},
|
||||
{"integer", "serial", true},
|
||||
{"INT4", "integer", true},
|
||||
{"varchar(10)", "varchar(20)", true},
|
||||
{"bigint", "serial", false},
|
||||
{"uuid", "bigint", false},
|
||||
{"", "bigint", true},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
if got := compatibleFKTypes(tt.a, tt.b); got != tt.want {
|
||||
t.Errorf("compatibleFKTypes(%q, %q) = %v, want %v", tt.a, tt.b, got, tt.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,137 @@
|
||||
package dbml
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"git.warky.dev/wdevs/relspecgo/pkg/models"
|
||||
)
|
||||
|
||||
// directiveLineRegex matches a dialect directive line:
|
||||
//
|
||||
// @postgres: partition by RANGE (created_at)
|
||||
// @postgres(id): identity always
|
||||
//
|
||||
// Group 1 is the namespace, group 2 the optional (column) target, group 3 the
|
||||
// raw argument text (validated separately so error messages can be specific).
|
||||
var directiveLineRegex = regexp.MustCompile(`^@([^():]*)(?:\(([^()]*)\))?\s*:(.*)$`)
|
||||
|
||||
// namespaceRegex is the grammar for a directive namespace.
|
||||
var namespaceRegex = regexp.MustCompile(`^[a-z][a-z0-9_]*$`)
|
||||
|
||||
// parsedDirective is a directive line that has been parsed but not yet attached
|
||||
// to a model object.
|
||||
type parsedDirective struct {
|
||||
namespace string
|
||||
target string // column name; "" when absent
|
||||
args string
|
||||
line int
|
||||
}
|
||||
|
||||
// parseDirectiveLine parses a single "@namespace[(target)]: args" line.
|
||||
func parseDirectiveLine(line string, lineNo int) (parsedDirective, error) {
|
||||
m := directiveLineRegex.FindStringSubmatch(line)
|
||||
if m == nil {
|
||||
return parsedDirective{}, fmt.Errorf(
|
||||
"dbml: line %d: malformed directive %q (expected \"@namespace: args\")", lineNo, line)
|
||||
}
|
||||
|
||||
ns := strings.TrimSpace(m[1])
|
||||
target := strings.TrimSpace(m[2])
|
||||
args := strings.TrimSpace(m[3])
|
||||
|
||||
if !namespaceRegex.MatchString(ns) {
|
||||
return parsedDirective{}, fmt.Errorf(
|
||||
"dbml: line %d: invalid directive namespace %q (must match [a-z][a-z0-9_]*)", lineNo, ns)
|
||||
}
|
||||
if args == "" {
|
||||
return parsedDirective{}, fmt.Errorf("dbml: line %d: directive @%s has no arguments", lineNo, ns)
|
||||
}
|
||||
if target != "" {
|
||||
target = stripQuotes(target)
|
||||
}
|
||||
|
||||
return parsedDirective{namespace: ns, target: target, args: args, line: lineNo}, nil
|
||||
}
|
||||
|
||||
// attachDirective resolves the target model object from the current parser state
|
||||
// and stores the directive in its Metadata, enforcing location, duplicate and
|
||||
// strict-mode rules.
|
||||
func (r *Reader) attachDirective(
|
||||
pd parsedDirective,
|
||||
db *models.Database,
|
||||
table *models.Table,
|
||||
inTable, inIndexes bool,
|
||||
lastIndex *models.Index,
|
||||
) error {
|
||||
strict := r.options != nil && r.options.StrictDirectives
|
||||
key := models.DirectiveKey(pd.args)
|
||||
|
||||
var meta map[string]any
|
||||
var location string
|
||||
|
||||
switch {
|
||||
case inIndexes:
|
||||
if pd.target != "" {
|
||||
return fmt.Errorf("dbml: line %d: directive target (%s) is not allowed inside an indexes block", pd.line, pd.target)
|
||||
}
|
||||
if lastIndex == nil {
|
||||
return fmt.Errorf("dbml: line %d: directive @%s must follow an index definition", pd.line, pd.namespace)
|
||||
}
|
||||
if lastIndex.Metadata == nil {
|
||||
lastIndex.Metadata = make(map[string]any)
|
||||
}
|
||||
meta = lastIndex.Metadata
|
||||
location = models.DirectiveLocationIndex
|
||||
|
||||
case inTable && table != nil:
|
||||
if pd.target != "" {
|
||||
col, ok := table.Columns[pd.target]
|
||||
if !ok {
|
||||
return fmt.Errorf("dbml: line %d: directive target column %q not found in table %q", pd.line, pd.target, table.Name)
|
||||
}
|
||||
if col.Metadata == nil {
|
||||
col.Metadata = make(map[string]any)
|
||||
}
|
||||
meta = col.Metadata
|
||||
location = models.DirectiveLocationColumn
|
||||
} else {
|
||||
if table.Metadata == nil {
|
||||
table.Metadata = make(map[string]any)
|
||||
}
|
||||
meta = table.Metadata
|
||||
location = models.DirectiveLocationTable
|
||||
}
|
||||
|
||||
default:
|
||||
if pd.target != "" {
|
||||
return fmt.Errorf("dbml: line %d: directive target (%s) is only valid inside a table", pd.line, pd.target)
|
||||
}
|
||||
if db.Metadata == nil {
|
||||
db.Metadata = make(map[string]any)
|
||||
}
|
||||
meta = db.Metadata
|
||||
location = models.DirectiveLocationDatabase
|
||||
}
|
||||
|
||||
spec, documented := models.LookupDirectiveSpec(pd.namespace, key)
|
||||
|
||||
if strict && !documented {
|
||||
return fmt.Errorf("dbml: line %d: unknown directive @%s: %s (strict mode)", pd.line, pd.namespace, key)
|
||||
}
|
||||
if documented && !models.DirectiveLocationAllowed(pd.namespace, key, location) {
|
||||
return fmt.Errorf("dbml: line %d: directive @%s: %s is not valid at %s level", pd.line, pd.namespace, key, location)
|
||||
}
|
||||
if documented && spec.Singleton && models.HasDirective(meta, pd.namespace, key) {
|
||||
return fmt.Errorf("dbml: line %d: duplicate @%s directive %q at %s level", pd.line, pd.namespace, key, location)
|
||||
}
|
||||
|
||||
models.AddDirective(meta, models.Directive{
|
||||
Namespace: pd.namespace,
|
||||
Key: key,
|
||||
Args: pd.args,
|
||||
Line: pd.line,
|
||||
})
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,181 @@
|
||||
package dbml
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.warky.dev/wdevs/relspecgo/pkg/models"
|
||||
"git.warky.dev/wdevs/relspecgo/pkg/readers"
|
||||
)
|
||||
|
||||
func parse(t *testing.T, strict bool, src string) (*models.Database, error) {
|
||||
t.Helper()
|
||||
r := NewReader(&readers.ReaderOptions{StrictDirectives: strict})
|
||||
return r.parseDBML(src)
|
||||
}
|
||||
|
||||
func firstTable(t *testing.T, db *models.Database) *models.Table {
|
||||
t.Helper()
|
||||
if len(db.Schemas) == 0 || len(db.Schemas[0].Tables) == 0 {
|
||||
t.Fatal("no table parsed")
|
||||
}
|
||||
return db.Schemas[0].Tables[0]
|
||||
}
|
||||
|
||||
func TestDirectives_AttachAtEachLocation(t *testing.T) {
|
||||
src := `@postgres: search_path myapp
|
||||
|
||||
Table myapp.events {
|
||||
id bigint [pk]
|
||||
created_at timestamp [not null]
|
||||
@postgres(id): identity always
|
||||
@postgres: partition by RANGE (created_at)
|
||||
|
||||
indexes {
|
||||
(created_at) [name: 'idx_events_created']
|
||||
@postgres: with (fillfactor=90)
|
||||
}
|
||||
}
|
||||
`
|
||||
db, err := parse(t, false, src)
|
||||
if err != nil {
|
||||
t.Fatalf("parse: %v", err)
|
||||
}
|
||||
|
||||
if !models.HasDirective(db.Metadata, "postgres", "search_path") {
|
||||
t.Errorf("database-level directive missing: %+v", db.Metadata)
|
||||
}
|
||||
|
||||
tbl := firstTable(t, db)
|
||||
if !models.HasDirective(tbl.Metadata, "postgres", "partition") {
|
||||
t.Errorf("table-level directive missing: %+v", tbl.Metadata)
|
||||
}
|
||||
|
||||
col := tbl.Columns["id"]
|
||||
if col == nil || !models.HasDirective(col.Metadata, "postgres", "identity") {
|
||||
t.Errorf("column-level directive missing")
|
||||
}
|
||||
// Verbatim args preserved.
|
||||
if d := models.DirectivesForNamespace(col.Metadata, "postgres"); len(d) != 1 || d[0].Args != "identity always" {
|
||||
t.Errorf("column directive args = %+v", d)
|
||||
}
|
||||
|
||||
var idx *models.Index
|
||||
for _, i := range tbl.Indexes {
|
||||
idx = i
|
||||
}
|
||||
if idx == nil || !models.HasDirective(idx.Metadata, "postgres", "with") {
|
||||
t.Errorf("index-level directive missing: %+v", idx)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDirectives_RepeatablePreservedAndOrdered(t *testing.T) {
|
||||
src := `Table s.t {
|
||||
id int [pk]
|
||||
@postgres: with (fillfactor=90)
|
||||
@postgres: with (autovacuum_enabled=off)
|
||||
}
|
||||
`
|
||||
db, err := parse(t, false, src)
|
||||
if err != nil {
|
||||
t.Fatalf("parse: %v", err)
|
||||
}
|
||||
tbl := firstTable(t, db)
|
||||
got := models.DirectivesForNamespace(tbl.Metadata, "postgres")
|
||||
if len(got) != 2 {
|
||||
t.Fatalf("got %d directives, want 2", len(got))
|
||||
}
|
||||
if got[0].Args != "with (fillfactor=90)" || got[1].Args != "with (autovacuum_enabled=off)" {
|
||||
t.Errorf("repeatable directives out of order: %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDirectives_SingletonDuplicateErrors(t *testing.T) {
|
||||
src := `Table s.t {
|
||||
id int [pk]
|
||||
@postgres: partition by RANGE (a)
|
||||
@postgres: partition by LIST (b)
|
||||
}
|
||||
`
|
||||
_, err := parse(t, false, src)
|
||||
if err == nil || !strings.Contains(err.Error(), "duplicate") {
|
||||
t.Fatalf("want duplicate error, got %v", err)
|
||||
}
|
||||
if !strings.Contains(err.Error(), "line 4") {
|
||||
t.Errorf("error not line-numbered: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDirectives_MalformedErrors(t *testing.T) {
|
||||
cases := map[string]string{
|
||||
"no colon": "@postgres partition by x",
|
||||
"empty args": "@postgres:",
|
||||
"bad namespace": "@Postgres: partition by x",
|
||||
"numeric prefix": "@1x: foo",
|
||||
}
|
||||
for name, line := range cases {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
src := "Table s.t {\n id int [pk]\n " + line + "\n}\n"
|
||||
_, err := parse(t, false, src)
|
||||
if err == nil {
|
||||
t.Fatalf("want error for %q", line)
|
||||
}
|
||||
if !strings.Contains(err.Error(), "line 3") {
|
||||
t.Errorf("error not line-numbered: %v", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestDirectives_UnknownPreservedNonStrict(t *testing.T) {
|
||||
src := `Table s.t {
|
||||
id int [pk]
|
||||
@postgres: frobnicate all the things
|
||||
@clickhouse: engine MergeTree
|
||||
}
|
||||
`
|
||||
db, err := parse(t, false, src)
|
||||
if err != nil {
|
||||
t.Fatalf("parse: %v", err)
|
||||
}
|
||||
tbl := firstTable(t, db)
|
||||
if !models.HasDirective(tbl.Metadata, "postgres", "frobnicate") {
|
||||
t.Error("unknown postgres key not preserved")
|
||||
}
|
||||
if !models.HasDirective(tbl.Metadata, "clickhouse", "engine") {
|
||||
t.Error("unknown namespace not preserved")
|
||||
}
|
||||
}
|
||||
|
||||
func TestDirectives_StrictErrors(t *testing.T) {
|
||||
src := `Table s.t {
|
||||
id int [pk]
|
||||
@postgres: frobnicate x
|
||||
}
|
||||
`
|
||||
_, err := parse(t, true, src)
|
||||
if err == nil || !strings.Contains(err.Error(), "strict mode") {
|
||||
t.Fatalf("want strict-mode error, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDirectives_UnknownColumnTargetErrors(t *testing.T) {
|
||||
src := `Table s.t {
|
||||
id int [pk]
|
||||
@postgres(missing): identity always
|
||||
}
|
||||
`
|
||||
_, err := parse(t, false, src)
|
||||
if err == nil || !strings.Contains(err.Error(), "not found") {
|
||||
t.Fatalf("want unknown-column error, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDirectives_WrongLocationErrors(t *testing.T) {
|
||||
// partition is table-only.
|
||||
src := "@postgres: partition by RANGE (x)\n\nTable s.t {\n id int [pk]\n}\n"
|
||||
_, err := parse(t, false, src)
|
||||
if err == nil || !strings.Contains(err.Error(), "not valid at database level") {
|
||||
t.Fatalf("want location error, got %v", err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,38 @@
|
||||
package dbml
|
||||
|
||||
import (
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"git.warky.dev/wdevs/relspecgo/pkg/models"
|
||||
)
|
||||
|
||||
var (
|
||||
generatedNoteRegex = regexp.MustCompile(`(?is)^GENERATED\s+ALWAYS\s+AS\s*\((.*)\)\s*STORED$`)
|
||||
identityNoteRegex = regexp.MustCompile(`(?i)^GENERATED\s+(ALWAYS|BY\s+DEFAULT)\s+AS\s+IDENTITY$`)
|
||||
)
|
||||
|
||||
// applyColumnMarkerNote interprets a column note written by the DBML writer for
|
||||
// GENERATED ... STORED and GENERATED ... AS IDENTITY columns, setting the matching
|
||||
// model fields. It returns true when the note was a marker, in which case it is not
|
||||
// a user comment.
|
||||
func applyColumnMarkerNote(column *models.Column, rawNote string) bool {
|
||||
note := strings.TrimSpace(rawNote)
|
||||
if len(note) >= 2 && (note[0] == '\'' || note[0] == '"') && note[len(note)-1] == note[0] {
|
||||
note = note[1 : len(note)-1]
|
||||
}
|
||||
note = strings.ReplaceAll(strings.ReplaceAll(note, `\'`, `'`), `\\`, `\`)
|
||||
note = strings.TrimSpace(note)
|
||||
|
||||
if m := generatedNoteRegex.FindStringSubmatch(note); m != nil {
|
||||
column.Generated = true
|
||||
column.GenerationExpression = strings.TrimSpace(m[1])
|
||||
return true
|
||||
}
|
||||
if m := identityNoteRegex.FindStringSubmatch(note); m != nil {
|
||||
column.Identity = true
|
||||
column.IdentityGeneration = strings.Join(strings.Fields(strings.ToUpper(m[1])), " ")
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
+223
-30
@@ -48,7 +48,22 @@ func (r *Reader) ReadDatabase() (*models.Database, error) {
|
||||
return nil, fmt.Errorf("failed to read file: %w", err)
|
||||
}
|
||||
|
||||
return r.parseDBML(string(content))
|
||||
db, err := r.parseDBML(string(content))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
r.resolveCommentedRefs(db)
|
||||
return db, nil
|
||||
}
|
||||
|
||||
// resolveCommentedRefs resolves the commented refs whose tables are loaded.
|
||||
// Unmatched refs stay pending for a later pass over a combined model.
|
||||
func (r *Reader) resolveCommentedRefs(db *models.Database) {
|
||||
for _, w := range ResolveCommentedRefs(db, false) {
|
||||
if r.options.Progress != nil {
|
||||
r.options.Progress("warning: " + w)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ReadSchema reads and parses DBML input, returning a Schema model
|
||||
@@ -125,6 +140,7 @@ func (r *Reader) readDirectoryDBML(dirPath string) (*models.Database, error) {
|
||||
}
|
||||
}
|
||||
|
||||
r.resolveCommentedRefs(db)
|
||||
return db, nil
|
||||
}
|
||||
|
||||
@@ -305,9 +321,39 @@ func sortDBMLFiles(files []string) []string {
|
||||
// Merges: Columns (map), Constraints (map), Indexes (map), Relationships (map)
|
||||
// Uses first non-empty Description
|
||||
func mergeTable(baseTable, fileTable *models.Table) {
|
||||
// Merge columns (map naturally merges - later keys overwrite)
|
||||
for key, col := range fileTable.Columns {
|
||||
baseTable.Columns[key] = col
|
||||
// Merge columns. Each file numbers its own columns from 1, so a table split
|
||||
// across files would otherwise end up with colliding Column.Sequence values
|
||||
// and writers would fall back to alphabetical order. Re-base the incoming
|
||||
// file's new columns after the highest sequence already present, preserving
|
||||
// their in-file order. Columns that overwrite an existing key keep the
|
||||
// original position.
|
||||
var maxSeq uint
|
||||
for _, col := range baseTable.Columns {
|
||||
if col.Sequence > maxSeq {
|
||||
maxSeq = col.Sequence
|
||||
}
|
||||
}
|
||||
|
||||
incoming := make([]*models.Column, 0, len(fileTable.Columns))
|
||||
for _, col := range fileTable.Columns {
|
||||
incoming = append(incoming, col)
|
||||
}
|
||||
sort.Slice(incoming, func(i, j int) bool {
|
||||
if incoming[i].Sequence != incoming[j].Sequence {
|
||||
return incoming[i].Sequence < incoming[j].Sequence
|
||||
}
|
||||
return incoming[i].Name < incoming[j].Name
|
||||
})
|
||||
|
||||
var added uint
|
||||
for _, col := range incoming {
|
||||
if existing, ok := baseTable.Columns[col.Name]; ok {
|
||||
col.Sequence = existing.Sequence
|
||||
} else {
|
||||
added++
|
||||
col.Sequence = maxSeq + added
|
||||
}
|
||||
baseTable.Columns[col.Name] = col
|
||||
}
|
||||
|
||||
// Merge constraints
|
||||
@@ -410,6 +456,10 @@ func mergeDatabase(baseDB, fileDB *models.Database) {
|
||||
// Merge domains
|
||||
baseDB.Domains = append(baseDB.Domains, fileDB.Domains...)
|
||||
|
||||
for _, ref := range PendingCommentedRefs(fileDB) {
|
||||
addPendingCommentedRef(baseDB, ref)
|
||||
}
|
||||
|
||||
// Use first non-empty description
|
||||
if baseDB.Description == "" && fileDB.Description != "" {
|
||||
baseDB.Description = fileDB.Description
|
||||
@@ -434,16 +484,55 @@ func (r *Reader) parseDBML(content string) (*models.Database, error) {
|
||||
var currentSchema string
|
||||
var inIndexes bool
|
||||
var inTable bool
|
||||
var inTableNote bool
|
||||
var tableNoteLines []string
|
||||
tableNoteStartLine := 0
|
||||
var columnSeq uint
|
||||
var lastIndex *models.Index // most recent index in the current Indexes block
|
||||
lineNo := 0
|
||||
|
||||
tableRegex := regexp.MustCompile(`^Table\s+(.+?)\s*{`)
|
||||
refRegex := regexp.MustCompile(`^Ref:\s+(.+)`)
|
||||
|
||||
for scanner.Scan() {
|
||||
line := strings.TrimSpace(scanner.Text())
|
||||
lineNo++
|
||||
rawLine := scanner.Text()
|
||||
line := strings.TrimSpace(rawLine)
|
||||
|
||||
// Skip empty lines and comments
|
||||
// A table note can use DBML's triple-quoted form. Its contents must be
|
||||
// consumed before normal parsing, otherwise each prose line is mistaken
|
||||
// for a column declaration.
|
||||
if inTableNote {
|
||||
if line == "'''" {
|
||||
setTableNote(currentTable, strings.TrimSpace(strings.Join(tableNoteLines, "\n")))
|
||||
inTableNote = false
|
||||
tableNoteLines = nil
|
||||
continue
|
||||
}
|
||||
tableNoteLines = append(tableNoteLines, strings.TrimSpace(rawLine))
|
||||
continue
|
||||
}
|
||||
|
||||
// Skip empty lines and comments. A commented `// Ref:` is kept as a
|
||||
// pending cross-file ref, resolved once every file is loaded.
|
||||
if line == "" || strings.HasPrefix(line, "//") {
|
||||
if ref, ok := commentedRef(line); ok {
|
||||
addPendingCommentedRef(db, ref)
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
// Parse a dialect directive (@postgres:, @sqlite:, …). Handled before
|
||||
// table/column/index parsing so directive lines are never mistaken for
|
||||
// columns.
|
||||
if strings.HasPrefix(line, "@") {
|
||||
pd, err := parseDirectiveLine(line, lineNo)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := r.attachDirective(pd, db, currentTable, inTable, inIndexes, lastIndex); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
@@ -474,8 +563,10 @@ func (r *Reader) parseDBML(content string) (*models.Database, error) {
|
||||
continue
|
||||
}
|
||||
|
||||
// End of table definition
|
||||
if inTable && line == "}" {
|
||||
// End of table definition. Guarded by !inIndexes so the closing brace
|
||||
// of an `indexes { }` block is not mistaken for the end of the table
|
||||
// (which would drop any table-level content that follows it).
|
||||
if inTable && !inIndexes && line == "}" {
|
||||
if currentTable != nil && currentSchema != "" {
|
||||
schemaMap[currentSchema].Tables = append(schemaMap[currentSchema].Tables, currentTable)
|
||||
currentTable = nil
|
||||
@@ -488,12 +579,14 @@ func (r *Reader) parseDBML(content string) (*models.Database, error) {
|
||||
// Parse indexes section
|
||||
if inTable && (strings.HasPrefix(line, "Indexes {") || strings.HasPrefix(line, "indexes {")) {
|
||||
inIndexes = true
|
||||
lastIndex = nil
|
||||
continue
|
||||
}
|
||||
|
||||
// End of indexes section
|
||||
if inIndexes && line == "}" {
|
||||
inIndexes = false
|
||||
lastIndex = nil
|
||||
continue
|
||||
}
|
||||
|
||||
@@ -512,16 +605,30 @@ func (r *Reader) parseDBML(content string) (*models.Database, error) {
|
||||
|
||||
index := r.parseIndex(line, currentTable.Name, currentSchema)
|
||||
if index != nil {
|
||||
currentTable.Indexes[index.Name] = index
|
||||
// Keep a reused name under a unique map key so the duplicate is
|
||||
// not silently dropped; the inspector reports it.
|
||||
key := index.Name
|
||||
for n := 2; currentTable.Indexes[key] != nil; n++ {
|
||||
key = fmt.Sprintf("%s#%d", index.Name, n)
|
||||
}
|
||||
currentTable.Indexes[key] = index
|
||||
lastIndex = index
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
// Parse table note
|
||||
if inTable && currentTable != nil && strings.HasPrefix(line, "Note:") {
|
||||
note := strings.TrimPrefix(line, "Note:")
|
||||
// Parse table note. DBML files in the wild use both `Note:` and
|
||||
// `note:`, so accept either spelling.
|
||||
if inTable && currentTable != nil && strings.HasPrefix(strings.ToLower(line), "note:") {
|
||||
note := strings.TrimSpace(line[len("note:"):])
|
||||
if strings.TrimSpace(note) == "'''" {
|
||||
inTableNote = true
|
||||
tableNoteLines = nil
|
||||
tableNoteStartLine = lineNo
|
||||
continue
|
||||
}
|
||||
note = strings.Trim(note, " '\"")
|
||||
currentTable.Description = note
|
||||
setTableNote(currentTable, note)
|
||||
continue
|
||||
}
|
||||
|
||||
@@ -558,6 +665,13 @@ func (r *Reader) parseDBML(content string) (*models.Database, error) {
|
||||
}
|
||||
}
|
||||
|
||||
if err := scanner.Err(); err != nil {
|
||||
return nil, fmt.Errorf("failed to scan DBML: %w", err)
|
||||
}
|
||||
if inTableNote {
|
||||
return nil, fmt.Errorf("dbml: line %d: unterminated triple-quoted table note", tableNoteStartLine)
|
||||
}
|
||||
|
||||
// Assign pending constraints to their respective tables
|
||||
for _, constraint := range pendingConstraints {
|
||||
// Find the table this constraint belongs to
|
||||
@@ -575,20 +689,12 @@ func (r *Reader) parseDBML(content string) (*models.Database, error) {
|
||||
// for DBML refs so diffing equivalent schemas compares the same model.
|
||||
for _, schema := range schemaMap {
|
||||
for _, table := range schema.Tables {
|
||||
for _, constraint := range table.Constraints {
|
||||
for _, name := range sortedConstraintNames(table.Constraints) {
|
||||
constraint := table.Constraints[name]
|
||||
if constraint.Type != models.ForeignKeyConstraint {
|
||||
continue
|
||||
}
|
||||
name := fmt.Sprintf("%s_to_%s", table.Name, constraint.ReferencedTable)
|
||||
relationship := models.InitRelationship(name, models.OneToMany)
|
||||
relationship.FromTable = table.Name
|
||||
relationship.FromSchema = table.Schema
|
||||
relationship.FromColumns = append([]string(nil), constraint.Columns...)
|
||||
relationship.ToTable = constraint.ReferencedTable
|
||||
relationship.ToSchema = constraint.ReferencedSchema
|
||||
relationship.ToColumns = append([]string(nil), constraint.ReferencedColumns...)
|
||||
relationship.ForeignKey = constraint.Name
|
||||
table.Relationships[name] = relationship
|
||||
addFKRelationship(table, constraint)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -601,6 +707,53 @@ func (r *Reader) parseDBML(content string) (*models.Database, error) {
|
||||
return db, nil
|
||||
}
|
||||
|
||||
// sortedConstraintNames returns constraint keys in sorted order so derived
|
||||
// relationship names do not depend on map iteration order.
|
||||
func sortedConstraintNames(constraints map[string]*models.Constraint) []string {
|
||||
names := make([]string, 0, len(constraints))
|
||||
for name := range constraints {
|
||||
names = append(names, name)
|
||||
}
|
||||
sort.Strings(names)
|
||||
return names
|
||||
}
|
||||
|
||||
// addFKRelationship derives the relationship for a foreign key, matching how
|
||||
// the PostgreSQL reader models FKs.
|
||||
func addFKRelationship(table *models.Table, constraint *models.Constraint) {
|
||||
name := fmt.Sprintf("%s_to_%s", table.Name, constraint.ReferencedTable)
|
||||
if existing, taken := table.Relationships[name]; taken && existing.ForeignKey != constraint.Name {
|
||||
// A second FK to the same table must not overwrite the first.
|
||||
name = fmt.Sprintf("%s_%s", name, strings.Join(constraint.Columns, "_"))
|
||||
}
|
||||
relationship := models.InitRelationship(name, models.OneToMany)
|
||||
relationship.FromTable = table.Name
|
||||
relationship.FromSchema = table.Schema
|
||||
relationship.FromColumns = append([]string(nil), constraint.Columns...)
|
||||
relationship.ToTable = constraint.ReferencedTable
|
||||
relationship.ToSchema = constraint.ReferencedSchema
|
||||
relationship.ToColumns = append([]string(nil), constraint.ReferencedColumns...)
|
||||
relationship.ForeignKey = constraint.Name
|
||||
if table.Relationships == nil {
|
||||
table.Relationships = make(map[string]*models.Relationship)
|
||||
}
|
||||
table.Relationships[name] = relationship
|
||||
}
|
||||
|
||||
// setTableNote preserves multiple table notes. The first maps to Description
|
||||
// and the second to Comment, matching the model fields used by code writers.
|
||||
func setTableNote(table *models.Table, note string) {
|
||||
if table.Description == "" {
|
||||
table.Description = note
|
||||
return
|
||||
}
|
||||
if table.Comment == "" {
|
||||
table.Comment = note
|
||||
return
|
||||
}
|
||||
table.Comment += "\n" + note
|
||||
}
|
||||
|
||||
// parseColumn parses a DBML column definition
|
||||
func (r *Reader) parseColumn(line, tableName, schemaName string) (*models.Column, *models.Constraint) {
|
||||
// Format: column_name type [attributes] // comment
|
||||
@@ -618,7 +771,7 @@ func (r *Reader) parseColumn(line, tableName, schemaName string) (*models.Column
|
||||
|
||||
// Parse attributes in brackets
|
||||
if attrs != "" {
|
||||
attrList := strings.Split(attrs, ",")
|
||||
attrList := splitColumnAttrs(attrs)
|
||||
|
||||
for _, attr := range attrList {
|
||||
attr = strings.TrimSpace(attr)
|
||||
@@ -651,7 +804,9 @@ func (r *Reader) parseColumn(line, tableName, schemaName string) (*models.Column
|
||||
} else if strings.HasPrefix(attr, "note:") {
|
||||
// Parse column note/comment
|
||||
note := strings.TrimSpace(strings.TrimPrefix(attr, "note:"))
|
||||
column.Comment = strings.Trim(note, "'\"")
|
||||
if !applyColumnMarkerNote(column, note) {
|
||||
column.Comment = strings.Trim(note, "'\"")
|
||||
}
|
||||
} else if strings.HasPrefix(attr, "ref:") {
|
||||
// Parse inline reference
|
||||
// DBML semantics depend on context:
|
||||
@@ -701,7 +856,45 @@ func (r *Reader) parseColumn(line, tableName, schemaName string) (*models.Column
|
||||
return column, constraint
|
||||
}
|
||||
|
||||
func splitInlineComment(line string) (content string, inlineComment string) {
|
||||
// splitColumnAttrs splits a DBML attribute list on top-level commas. Notes and
|
||||
// quoted defaults may contain commas of their own, which are part of the value
|
||||
// rather than attribute separators.
|
||||
func splitColumnAttrs(attrs string) []string {
|
||||
var result []string
|
||||
start := 0
|
||||
var quote byte
|
||||
escaped := false
|
||||
|
||||
for i := 0; i < len(attrs); i++ {
|
||||
ch := attrs[i]
|
||||
if quote != 0 {
|
||||
if escaped {
|
||||
escaped = false
|
||||
continue
|
||||
}
|
||||
if ch == '\\' {
|
||||
escaped = true
|
||||
continue
|
||||
}
|
||||
if ch == quote {
|
||||
quote = 0
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
switch ch {
|
||||
case '\'', '"', '`':
|
||||
quote = ch
|
||||
case ',':
|
||||
result = append(result, attrs[start:i])
|
||||
start = i + 1
|
||||
}
|
||||
}
|
||||
|
||||
return append(result, attrs[start:])
|
||||
}
|
||||
|
||||
func splitInlineComment(line string) (content, inlineComment string) {
|
||||
commentStart := strings.Index(line, "//")
|
||||
if commentStart == -1 {
|
||||
return line, ""
|
||||
@@ -710,7 +903,7 @@ func splitInlineComment(line string) (content string, inlineComment string) {
|
||||
return strings.TrimSpace(line[:commentStart]), strings.TrimSpace(line[commentStart+2:])
|
||||
}
|
||||
|
||||
func splitColumnSignatureAndAttrs(line string) (signature string, attrs string) {
|
||||
func splitColumnSignatureAndAttrs(line string) (signature, attrs string) {
|
||||
trimmed := strings.TrimSpace(line)
|
||||
if trimmed == "" || !strings.HasSuffix(trimmed, "]") {
|
||||
return trimmed, ""
|
||||
@@ -736,7 +929,7 @@ func splitColumnSignatureAndAttrs(line string) (signature string, attrs string)
|
||||
return trimmed, ""
|
||||
}
|
||||
|
||||
func parseColumnSignature(signature string) (columnName string, columnType string, ok bool) {
|
||||
func parseColumnSignature(signature string) (columnName, columnType string, ok bool) {
|
||||
signature = strings.TrimSpace(signature)
|
||||
if signature == "" {
|
||||
return "", "", false
|
||||
@@ -1041,5 +1234,5 @@ func (r *Reader) parseTableRef(ref string) (schema, table string, columns []stri
|
||||
table = stripQuotes(parts[0])
|
||||
}
|
||||
|
||||
return
|
||||
return schema, table, columns
|
||||
}
|
||||
|
||||
@@ -652,6 +652,22 @@ func TestReadDirectory_TableMerging(t *testing.T) {
|
||||
if emailCol.Type != "varchar(255)" {
|
||||
t.Errorf("Expected email type 'varchar(255)', got '%s'", emailCol.Type)
|
||||
}
|
||||
|
||||
// Merged columns must keep declaration order (file 1: id, email; file 3:
|
||||
// name, created_at) via strictly increasing, non-colliding Sequence values
|
||||
// so downstream writers do not fall back to alphabetical order.
|
||||
order := []string{"id", "email", "name", "created_at"}
|
||||
var prev uint
|
||||
for i, name := range order {
|
||||
col := usersTable.Columns[name]
|
||||
if col.Sequence == 0 {
|
||||
t.Fatalf("column %q has zero Sequence after merge", name)
|
||||
}
|
||||
if i > 0 && col.Sequence <= prev {
|
||||
t.Errorf("column %q Sequence %d not greater than previous %d (order not preserved across files)", name, col.Sequence, prev)
|
||||
}
|
||||
prev = col.Sequence
|
||||
}
|
||||
}
|
||||
|
||||
func TestReadDirectory_CommentedRefsLast(t *testing.T) {
|
||||
@@ -689,7 +705,7 @@ func TestReadDirectory_CommentedRefsLast(t *testing.T) {
|
||||
func TestReadDirectory_EmptyDirectory(t *testing.T) {
|
||||
// Create a temporary empty directory
|
||||
tmpDir := filepath.Join("..", "..", "..", "tests", "assets", "dbml", "empty_test_dir")
|
||||
err := os.MkdirAll(tmpDir, 0755)
|
||||
err := os.MkdirAll(tmpDir, 0o755)
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to create temp directory: %v", err)
|
||||
}
|
||||
@@ -956,7 +972,7 @@ func TestReader_CompositePKIndex(t *testing.T) {
|
||||
`
|
||||
dir := t.TempDir()
|
||||
path := filepath.Join(dir, "composite_pk.dbml")
|
||||
if err := os.WriteFile(path, []byte(dbmlContent), 0644); err != nil {
|
||||
if err := os.WriteFile(path, []byte(dbmlContent), 0o644); err != nil {
|
||||
t.Fatalf("failed to write fixture: %v", err)
|
||||
}
|
||||
|
||||
@@ -1005,7 +1021,7 @@ func TestReader_ColumnPKOrderPreserved(t *testing.T) {
|
||||
`
|
||||
dir := t.TempDir()
|
||||
path := filepath.Join(dir, "column_pk_order.dbml")
|
||||
if err := os.WriteFile(path, []byte(dbmlContent), 0644); err != nil {
|
||||
if err := os.WriteFile(path, []byte(dbmlContent), 0o644); err != nil {
|
||||
t.Fatalf("failed to write fixture: %v", err)
|
||||
}
|
||||
|
||||
@@ -1033,3 +1049,75 @@ func TestReader_ColumnPKOrderPreserved(t *testing.T) {
|
||||
t.Errorf("expected snapshot_id (declared first) to have a lower Sequence than artifact_id, got %d >= %d", snapshotCol.Sequence, artifactCol.Sequence)
|
||||
}
|
||||
}
|
||||
|
||||
func TestReader_MultilineTableNote(t *testing.T) {
|
||||
dbmlContent := "Table \"info\".\"city\" {\n" +
|
||||
" \"id_city\" serial [pk, not null, increment]\n" +
|
||||
" \"name\" text [not null, note: 'first, second, third']\n\n" +
|
||||
" note: '''\n" +
|
||||
" Cities and municipalities worldwide.\n\n" +
|
||||
" SPATIAL:\n" +
|
||||
" Proximity queries use a GiST index.\n" +
|
||||
" '''\n" +
|
||||
" Note: 'Short summary'\n" +
|
||||
"}\n"
|
||||
path := filepath.Join(t.TempDir(), "city.dbml")
|
||||
if err := os.WriteFile(path, []byte(dbmlContent), 0o644); err != nil {
|
||||
t.Fatalf("failed to write fixture: %v", err)
|
||||
}
|
||||
|
||||
db, err := NewReader(&readers.ReaderOptions{FilePath: path}).ReadDatabase()
|
||||
if err != nil {
|
||||
t.Fatalf("ReadDatabase() error = %v", err)
|
||||
}
|
||||
|
||||
table := db.Schemas[0].Tables[0]
|
||||
if got, want := table.Description, "Cities and municipalities worldwide.\n\nSPATIAL:\nProximity queries use a GiST index."; got != want {
|
||||
t.Errorf("table description = %q, want %q", got, want)
|
||||
}
|
||||
if got, want := table.Comment, "Short summary"; got != want {
|
||||
t.Errorf("table comment = %q, want %q", got, want)
|
||||
}
|
||||
if got, want := len(table.Columns), 2; got != want {
|
||||
t.Errorf("column count = %d, want %d; note body must not be parsed as columns", got, want)
|
||||
}
|
||||
if got, want := table.Columns["name"].Comment, "first, second, third"; got != want {
|
||||
t.Errorf("column note = %q, want %q", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
// A name reused inside one Indexes block must not silently drop the earlier
|
||||
// index; both are kept so the inspector can report the duplicate.
|
||||
func TestReader_DuplicateIndexNameInTableKept(t *testing.T) {
|
||||
dbmlContent := `Table individual_actor {
|
||||
id bigint [pk]
|
||||
rid_actor bigint
|
||||
kind text
|
||||
|
||||
Indexes {
|
||||
(rid_actor) [name: 'idx_actor', unique]
|
||||
(rid_actor, kind) [name: 'idx_actor']
|
||||
}
|
||||
}
|
||||
`
|
||||
dir := t.TempDir()
|
||||
path := filepath.Join(dir, "dup_index.dbml")
|
||||
if err := os.WriteFile(path, []byte(dbmlContent), 0o644); err != nil {
|
||||
t.Fatalf("failed to write fixture: %v", err)
|
||||
}
|
||||
|
||||
db, err := NewReader(&readers.ReaderOptions{FilePath: path}).ReadDatabase()
|
||||
if err != nil {
|
||||
t.Fatalf("ReadDatabase() error = %v", err)
|
||||
}
|
||||
|
||||
table := db.Schemas[0].Tables[0]
|
||||
if len(table.Indexes) != 2 {
|
||||
t.Fatalf("expected 2 indexes, got %d: %v", len(table.Indexes), table.Indexes)
|
||||
}
|
||||
for key, idx := range table.Indexes {
|
||||
if idx.Name != "idx_actor" {
|
||||
t.Errorf("index %q: Name = %q, want idx_actor", key, idx.Name)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -85,6 +85,7 @@ export const postsRelations = relations(posts, ({ one }) => ({
|
||||
|
||||
## Notes
|
||||
|
||||
- `.generatedAlwaysAs(sql`expr`)` sets `Generated` + `GenerationExpression`; `.generatedAlwaysAsIdentity()` / `.generatedByDefaultAsIdentity()` set `Identity` + `IdentityGeneration`
|
||||
- Supports both PostgreSQL and MySQL Drizzle schemas
|
||||
- Extracts relationship information from `relations` definitions
|
||||
- Schema defaults to `public` for PostgreSQL
|
||||
|
||||
@@ -495,9 +495,24 @@ func (r *Reader) parseColumnModifiers(line string, column *models.Column, table
|
||||
}
|
||||
}
|
||||
|
||||
// Check for .generatedAlwaysAsIdentity()
|
||||
// Check for .generatedAlwaysAsIdentity() / .generatedByDefaultAsIdentity()
|
||||
if strings.Contains(line, ".generatedAlwaysAsIdentity()") {
|
||||
column.AutoIncrement = true
|
||||
column.Identity = true
|
||||
column.IdentityGeneration = "ALWAYS"
|
||||
}
|
||||
if strings.Contains(line, ".generatedByDefaultAsIdentity()") {
|
||||
column.AutoIncrement = true
|
||||
column.Identity = true
|
||||
column.IdentityGeneration = "BY DEFAULT"
|
||||
}
|
||||
|
||||
// Check for .generatedAlwaysAs(sql`expr`) (generated column)
|
||||
if idx := strings.Index(line, ".generatedAlwaysAs("); idx != -1 {
|
||||
if expr, ok := parseGeneratedAlwaysAs(line[idx+len(".generatedAlwaysAs("):]); ok {
|
||||
column.Generated = true
|
||||
column.GenerationExpression = expr
|
||||
}
|
||||
}
|
||||
|
||||
// Check for .references(() => otherTable.column)
|
||||
@@ -615,3 +630,34 @@ func (r *Reader) varNameToTableName(varName string) string {
|
||||
// For now, assume variable name matches table name
|
||||
return varName
|
||||
}
|
||||
|
||||
// parseGeneratedAlwaysAs extracts the SQL expression from the argument list of
|
||||
// generatedAlwaysAs(...), which follows the opening parenthesis in rest. The expression is
|
||||
// the contents of a sql`...` template (backticks inside it are escaped with a backslash)
|
||||
// or, failing that, a quoted string.
|
||||
func parseGeneratedAlwaysAs(rest string) (string, bool) {
|
||||
rest = strings.TrimSpace(rest)
|
||||
rest = strings.TrimPrefix(rest, "sql")
|
||||
if rest == "" {
|
||||
return "", false
|
||||
}
|
||||
quote := rest[0]
|
||||
if quote != '`' && quote != '\'' && quote != '"' {
|
||||
return "", false
|
||||
}
|
||||
|
||||
var sb strings.Builder
|
||||
for i := 1; i < len(rest); i++ {
|
||||
ch := rest[i]
|
||||
if ch == '\\' && i+1 < len(rest) {
|
||||
i++
|
||||
sb.WriteByte(rest[i])
|
||||
continue
|
||||
}
|
||||
if ch == quote {
|
||||
return strings.TrimSpace(sb.String()), true
|
||||
}
|
||||
sb.WriteByte(ch)
|
||||
}
|
||||
return "", false
|
||||
}
|
||||
|
||||
@@ -78,6 +78,8 @@ The reader recognizes the following GORM struct tags:
|
||||
- `not null` - NOT NULL constraint
|
||||
- `autoIncrement` - Auto-increment column
|
||||
- `default` - Default value
|
||||
- `generated` - Generated column marker (`Generated`; expression is not in the tag)
|
||||
- `identity` - Identity column marker (`Identity`, `ALWAYS`)
|
||||
- `size` - Column size/length
|
||||
- `index` - Create index
|
||||
- `uniqueIndex` - Create unique index
|
||||
|
||||
@@ -0,0 +1,48 @@
|
||||
package gorm
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"git.warky.dev/wdevs/relspecgo/pkg/readers"
|
||||
)
|
||||
|
||||
func TestReader_GeneratedAndIdentityMarkers(t *testing.T) {
|
||||
src := `package models
|
||||
|
||||
type Person struct {
|
||||
ID int64 ` + "`gorm:\"column:id;primaryKey;autoIncrement;type:bigint;identity\"`" + `
|
||||
Seq int64 ` + "`gorm:\"column:seq;type:bigint;<-:false;identity;not null\"`" + `
|
||||
FullName string ` + "`gorm:\"column:full_name;type:text;<-:false;generated\"`" + `
|
||||
First string ` + "`gorm:\"column:first;type:text\"`" + `
|
||||
}
|
||||
|
||||
func (Person) TableName() string {
|
||||
return "people"
|
||||
}
|
||||
`
|
||||
path := filepath.Join(t.TempDir(), "person.go")
|
||||
if err := os.WriteFile(path, []byte(src), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
db, err := NewReader(&readers.ReaderOptions{FilePath: path}).ReadDatabase()
|
||||
if err != nil {
|
||||
t.Fatalf("ReadDatabase() error = %v", err)
|
||||
}
|
||||
cols := db.Schemas[0].Tables[0].Columns
|
||||
|
||||
if !cols["id"].Identity || cols["id"].IdentityGeneration != "ALWAYS" {
|
||||
t.Errorf("id: identity = %v/%q, want true/ALWAYS", cols["id"].Identity, cols["id"].IdentityGeneration)
|
||||
}
|
||||
if !cols["seq"].Identity {
|
||||
t.Error("seq should be identity")
|
||||
}
|
||||
if !cols["full_name"].Generated {
|
||||
t.Error("full_name should be generated")
|
||||
}
|
||||
if cols["first"].Generated || cols["first"].Identity {
|
||||
t.Error("first must be an ordinary column")
|
||||
}
|
||||
}
|
||||
@@ -246,7 +246,7 @@ func (r *Reader) getReceiverType(expr ast.Expr) string {
|
||||
}
|
||||
|
||||
// parseTableNameMethod parses a TableName() method and extracts the table and schema name
|
||||
func (r *Reader) parseTableNameMethod(funcDecl *ast.FuncDecl) (tableName string, schemaName string) {
|
||||
func (r *Reader) parseTableNameMethod(funcDecl *ast.FuncDecl) (tableName, schemaName string) {
|
||||
if funcDecl.Body == nil {
|
||||
return "", ""
|
||||
}
|
||||
@@ -669,7 +669,7 @@ func (r *Reader) parseIndexesFromTag(table *models.Table, column *models.Column,
|
||||
}
|
||||
|
||||
// extractTableFromGormTag extracts table and schema from gorm tag
|
||||
func (r *Reader) extractTableFromGormTag(tag string) (tablename string, schemaName string) {
|
||||
func (r *Reader) extractTableFromGormTag(tag string) (tablename, schemaName string) {
|
||||
// This is typically set via TableName() method, not in tags
|
||||
// We'll return empty strings and rely on deriveTableName
|
||||
return "", ""
|
||||
@@ -726,6 +726,15 @@ func (r *Reader) parseColumn(fieldName string, fieldType ast.Expr, tag string, s
|
||||
if _, ok := parts["autoincrement"]; ok {
|
||||
column.AutoIncrement = true
|
||||
}
|
||||
if _, ok := parts["generated"]; ok {
|
||||
// GENERATED ... STORED marker written by the GORM writer; the expression is not in the tag
|
||||
column.Generated = true
|
||||
}
|
||||
if _, ok := parts["identity"]; ok {
|
||||
// GENERATED ALWAYS AS IDENTITY marker written by the GORM writer
|
||||
column.Identity = true
|
||||
column.IdentityGeneration = "ALWAYS"
|
||||
}
|
||||
if def, ok := parts["default"]; ok {
|
||||
// Default value from GORM tag (e.g., default:gen_random_uuid())
|
||||
column.Default = def
|
||||
@@ -794,12 +803,12 @@ func (r *Reader) parseTypeWithLength(typeStr string) (baseType string, length in
|
||||
if pgsql.SupportsLength(rawBaseType) && !strings.Contains(parens, ",") {
|
||||
if _, err := fmt.Sscanf(parens, "%d", &length); err == nil {
|
||||
baseType = pgsql.CanonicalizeBaseType(rawBaseType)
|
||||
return
|
||||
return baseType, length
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return
|
||||
return baseType, length
|
||||
}
|
||||
|
||||
// parseTypeWithReferences parses a type string and extracts base type, length, and references
|
||||
@@ -816,12 +825,12 @@ func (r *Reader) parseTypeWithReferences(typeStr string) (baseType string, lengt
|
||||
|
||||
// Parse base type for length
|
||||
baseType, length = r.parseTypeWithLength(baseTypePart)
|
||||
return
|
||||
return baseType, length, refInfo
|
||||
}
|
||||
|
||||
// No references, just parse type and length
|
||||
baseType, length = r.parseTypeWithLength(typeStr)
|
||||
return
|
||||
return baseType, length, refInfo
|
||||
}
|
||||
|
||||
// parseGormTag parses a gorm tag string into a map
|
||||
|
||||
@@ -32,7 +32,7 @@ func (r *Reader) isScalarType(typeName string, ctx *parseContext) bool {
|
||||
return commonCustomScalars[typeName]
|
||||
}
|
||||
|
||||
func (r *Reader) graphQLTypeToSQL(gqlType string, fieldName string, typeName string) string {
|
||||
func (r *Reader) graphQLTypeToSQL(gqlType, fieldName, typeName string) string {
|
||||
// Check for ID type with configurable mapping
|
||||
if gqlType == "ID" {
|
||||
// Check metadata for ID type preference
|
||||
|
||||
@@ -25,6 +25,11 @@ sqlserver://user:pass@192.168.1.100:1433/production
|
||||
sqlserver://localhost/testdb?encrypt=disable
|
||||
```
|
||||
|
||||
## Computed and Identity Columns
|
||||
|
||||
- `sys.computed_columns` -> `Generated` + `GenerationExpression` (outer parentheses removed; persisted or not)
|
||||
- Identity columns -> `Identity` + `IdentityGeneration = ALWAYS`
|
||||
|
||||
## Supported Constraints
|
||||
|
||||
- Primary Keys
|
||||
|
||||
@@ -104,8 +104,12 @@ func (r *Reader) queryColumns(schemaName string) (map[string]map[string]*models.
|
||||
c.numeric_precision,
|
||||
c.numeric_scale,
|
||||
ISNULL(ep.value, '') as description,
|
||||
COLUMNPROPERTY(OBJECT_ID(QUOTENAME(c.table_schema) + '.' + QUOTENAME(c.table_name)), c.column_name, 'IsIdentity') as is_identity
|
||||
COLUMNPROPERTY(OBJECT_ID(QUOTENAME(c.table_schema) + '.' + QUOTENAME(c.table_name)), c.column_name, 'IsIdentity') as is_identity,
|
||||
cc.definition as computed_definition
|
||||
FROM information_schema.columns c
|
||||
LEFT JOIN sys.computed_columns cc
|
||||
ON cc.object_id = OBJECT_ID(QUOTENAME(c.table_schema) + '.' + QUOTENAME(c.table_name))
|
||||
AND cc.name = c.column_name
|
||||
LEFT JOIN sys.extended_properties ep
|
||||
ON ep.major_id = OBJECT_ID(QUOTENAME(c.table_schema) + '.' + QUOTENAME(c.table_name))
|
||||
AND ep.minor_id = COLUMNPROPERTY(OBJECT_ID(QUOTENAME(c.table_schema) + '.' + QUOTENAME(c.table_name)), c.column_name, 'ColumnId')
|
||||
@@ -127,8 +131,9 @@ func (r *Reader) queryColumns(schemaName string) (map[string]map[string]*models.
|
||||
var schema, tableName, columnName, isNullable, dataType, description string
|
||||
var ordinalPosition int
|
||||
var columnDefault, charMaxLength, numPrecision, numScale, isIdentity *int
|
||||
var computedDefinition *string
|
||||
|
||||
if err := rows.Scan(&schema, &tableName, &columnName, &ordinalPosition, &columnDefault, &isNullable, &dataType, &charMaxLength, &numPrecision, &numScale, &description, &isIdentity); err != nil {
|
||||
if err := rows.Scan(&schema, &tableName, &columnName, &ordinalPosition, &columnDefault, &isNullable, &dataType, &charMaxLength, &numPrecision, &numScale, &description, &isIdentity, &computedDefinition); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
@@ -144,6 +149,14 @@ func (r *Reader) queryColumns(schemaName string) (map[string]map[string]*models.
|
||||
// Check if this is an identity column (auto-increment)
|
||||
if isIdentity != nil && *isIdentity == 1 {
|
||||
column.AutoIncrement = true
|
||||
column.Identity = true
|
||||
column.IdentityGeneration = "ALWAYS"
|
||||
}
|
||||
|
||||
// Computed columns report their expression wrapped in an extra pair of parentheses
|
||||
if computedDefinition != nil && strings.TrimSpace(*computedDefinition) != "" {
|
||||
column.Generated = true
|
||||
column.GenerationExpression = trimOuterParens(*computedDefinition)
|
||||
}
|
||||
|
||||
if charMaxLength != nil && *charMaxLength > 0 {
|
||||
@@ -414,3 +427,25 @@ func (r *Reader) queryIndexes(schemaName string) (map[string][]*models.Index, er
|
||||
|
||||
return indexes, rows.Err()
|
||||
}
|
||||
|
||||
// trimOuterParens removes one pair of parentheses wrapping the whole expression, as
|
||||
// SQL Server stores computed column definitions.
|
||||
func trimOuterParens(expr string) string {
|
||||
expr = strings.TrimSpace(expr)
|
||||
if len(expr) < 2 || expr[0] != '(' || expr[len(expr)-1] != ')' {
|
||||
return expr
|
||||
}
|
||||
depth := 0
|
||||
for i := 0; i < len(expr); i++ {
|
||||
switch expr[i] {
|
||||
case '(':
|
||||
depth++
|
||||
case ')':
|
||||
depth--
|
||||
if depth == 0 && i != len(expr)-1 {
|
||||
return expr
|
||||
}
|
||||
}
|
||||
}
|
||||
return strings.TrimSpace(expr[1 : len(expr)-1])
|
||||
}
|
||||
|
||||
@@ -3,9 +3,10 @@ package mssql
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
|
||||
"git.warky.dev/wdevs/relspecgo/pkg/mssql"
|
||||
"git.warky.dev/wdevs/relspecgo/pkg/readers"
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
// TestMapDataType tests MSSQL type mapping to canonical types
|
||||
@@ -38,9 +39,9 @@ func TestMapDataType(t *testing.T) {
|
||||
// TestConvertCanonicalToMSSQL tests canonical to MSSQL type conversion
|
||||
func TestConvertCanonicalToMSSQL(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
canonicalType string
|
||||
expectedMSSQL string
|
||||
name string
|
||||
canonicalType string
|
||||
expectedMSSQL string
|
||||
}{
|
||||
{"int to INT", "int", "INT"},
|
||||
{"int64 to BIGINT", "int64", "BIGINT"},
|
||||
@@ -63,9 +64,9 @@ func TestConvertCanonicalToMSSQL(t *testing.T) {
|
||||
// TestConvertMSSQLToCanonical tests MSSQL to canonical type conversion
|
||||
func TestConvertMSSQLToCanonical(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
mssqlType string
|
||||
expectedType string
|
||||
name string
|
||||
mssqlType string
|
||||
expectedType string
|
||||
}{
|
||||
{"INT to int", "INT", "int"},
|
||||
{"BIGINT to int64", "BIGINT", "int64"},
|
||||
@@ -84,3 +85,18 @@ func TestConvertMSSQLToCanonical(t *testing.T) {
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestTrimOuterParens(t *testing.T) {
|
||||
tests := map[string]string{
|
||||
"(([a])+([b]))": "([a])+([b])",
|
||||
"([a]+[b])": "[a]+[b]",
|
||||
"([a])+([b])": "([a])+([b])",
|
||||
"[a]+[b]": "[a]+[b]",
|
||||
" (upper([a])) ": "upper([a])",
|
||||
}
|
||||
for in, want := range tests {
|
||||
if got := trimOuterParens(in); got != want {
|
||||
t.Errorf("trimOuterParens(%q) = %q, want %q", in, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,254 @@
|
||||
package mysql
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
_ "github.com/go-sql-driver/mysql"
|
||||
|
||||
"git.warky.dev/wdevs/relspecgo/pkg/mariadb"
|
||||
"git.warky.dev/wdevs/relspecgo/pkg/models"
|
||||
"git.warky.dev/wdevs/relspecgo/pkg/readers"
|
||||
)
|
||||
|
||||
type Reader struct {
|
||||
options *readers.ReaderOptions
|
||||
db *sql.DB
|
||||
ctx context.Context
|
||||
}
|
||||
|
||||
func NewReader(options *readers.ReaderOptions) *Reader {
|
||||
return &Reader{options: options, ctx: context.Background()}
|
||||
}
|
||||
|
||||
func (r *Reader) ReadDatabase() (*models.Database, error) {
|
||||
if r.options == nil || r.options.ConnectionString == "" {
|
||||
return nil, fmt.Errorf("connection string is required")
|
||||
}
|
||||
if err := r.connect(); err != nil {
|
||||
return nil, fmt.Errorf("failed to connect: %w", err)
|
||||
}
|
||||
defer r.close()
|
||||
var name, version string
|
||||
if err := r.db.QueryRowContext(r.ctx, "SELECT DATABASE()").Scan(&name); err != nil {
|
||||
return nil, fmt.Errorf("failed to get database name: %w", err)
|
||||
}
|
||||
_ = r.db.QueryRowContext(r.ctx, "SELECT VERSION()").Scan(&version)
|
||||
db := models.InitDatabase(name)
|
||||
db.DatabaseType, db.SourceFormat, db.DatabaseVersion = models.MySQLDatabaseType, "mysql", version
|
||||
schemas, err := r.querySchemas(name)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to query schemas: %w", err)
|
||||
}
|
||||
for _, schema := range schemas {
|
||||
tables, err := r.queryTables(schema.Name)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
schema.Tables = tables
|
||||
for _, table := range tables {
|
||||
table.Columns, err = r.queryColumns(schema.Name, table.Name)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
table.Constraints, err = r.queryConstraints(schema.Name, table.Name)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
table.Indexes, err = r.queryIndexes(schema.Name, table.Name)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
table.RefSchema = schema
|
||||
for _, c := range table.Constraints {
|
||||
if c.Type == models.ForeignKeyConstraint {
|
||||
r.deriveRelationship(table, c)
|
||||
}
|
||||
}
|
||||
}
|
||||
schema.RefDatabase = db
|
||||
db.Schemas = append(db.Schemas, schema)
|
||||
}
|
||||
return db, nil
|
||||
}
|
||||
|
||||
func (r *Reader) ReadSchema() (*models.Schema, error) {
|
||||
db, err := r.ReadDatabase()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(db.Schemas) == 0 {
|
||||
return nil, fmt.Errorf("no schemas found in database")
|
||||
}
|
||||
return db.Schemas[0], nil
|
||||
}
|
||||
|
||||
func (r *Reader) ReadTable() (*models.Table, error) {
|
||||
s, err := r.ReadSchema()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(s.Tables) == 0 {
|
||||
return nil, fmt.Errorf("no tables found in schema")
|
||||
}
|
||||
return s.Tables[0], nil
|
||||
}
|
||||
|
||||
func (r *Reader) connect() error {
|
||||
db, err := sql.Open("mysql", r.options.ConnectionString)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err = db.PingContext(r.ctx); err != nil {
|
||||
db.Close()
|
||||
return err
|
||||
}
|
||||
r.db = db
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *Reader) close() {
|
||||
if r.db != nil {
|
||||
_ = r.db.Close()
|
||||
}
|
||||
}
|
||||
func (r *Reader) mapDataType(t string) string { return mariadb.ConvertMariaDBToCanonical(t) }
|
||||
|
||||
func (r *Reader) querySchemas(current string) ([]*models.Schema, error) {
|
||||
rows, err := r.db.QueryContext(r.ctx, "SELECT SCHEMA_NAME FROM information_schema.SCHEMATA WHERE SCHEMA_NAME = ?", current)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
var out []*models.Schema
|
||||
for rows.Next() {
|
||||
var n string
|
||||
if err := rows.Scan(&n); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, models.InitSchema(n))
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
func (r *Reader) queryTables(schema string) ([]*models.Table, error) {
|
||||
rows, err := r.db.QueryContext(r.ctx, "SELECT TABLE_NAME FROM information_schema.TABLES WHERE TABLE_SCHEMA = ? AND TABLE_TYPE = 'BASE TABLE' ORDER BY TABLE_NAME", schema)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
var out []*models.Table
|
||||
for rows.Next() {
|
||||
var n string
|
||||
if err := rows.Scan(&n); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, models.InitTable(n, schema))
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
func (r *Reader) queryColumns(schema, table string) (map[string]*models.Column, error) {
|
||||
rows, err := r.db.QueryContext(r.ctx, `SELECT COLUMN_NAME, COLUMN_TYPE, IS_NULLABLE, COLUMN_DEFAULT, ORDINAL_POSITION, EXTRA, COLUMN_COMMENT FROM information_schema.COLUMNS WHERE TABLE_SCHEMA = ? AND TABLE_NAME = ? ORDER BY ORDINAL_POSITION`, schema, table)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
out := map[string]*models.Column{}
|
||||
for rows.Next() {
|
||||
var name, typ, nullable, extra, comment string
|
||||
var def sql.NullString
|
||||
var pos int
|
||||
if err := rows.Scan(&name, &typ, &nullable, &def, &pos, &extra, &comment); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
c := models.InitColumn(name, table, schema)
|
||||
c.Type = r.mapDataType(typ)
|
||||
c.NotNull = strings.EqualFold(nullable, "NO")
|
||||
c.Sequence = uint(pos)
|
||||
c.Comment = comment
|
||||
if def.Valid {
|
||||
c.Default = def.String
|
||||
}
|
||||
c.AutoIncrement = strings.Contains(strings.ToLower(extra), "auto_increment")
|
||||
out[name] = c
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
func (r *Reader) queryConstraints(schema, table string) (map[string]*models.Constraint, error) {
|
||||
rows, err := r.db.QueryContext(r.ctx, `SELECT CONSTRAINT_NAME, CONSTRAINT_TYPE, COLUMN_NAME, REFERENCED_TABLE_SCHEMA, REFERENCED_TABLE_NAME, REFERENCED_COLUMN_NAME, ORDINAL_POSITION FROM information_schema.KEY_COLUMN_USAGE k JOIN information_schema.TABLE_CONSTRAINTS t USING (CONSTRAINT_SCHEMA, TABLE_NAME, CONSTRAINT_NAME) WHERE k.TABLE_SCHEMA=? AND k.TABLE_NAME=? ORDER BY CONSTRAINT_NAME, ORDINAL_POSITION`, schema, table)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
out := map[string]*models.Constraint{}
|
||||
for rows.Next() {
|
||||
var name, typ, col, rs, rt, rc string
|
||||
var pos int
|
||||
if err := rows.Scan(&name, &typ, &col, &rs, &rt, &rc, &pos); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
c := out[name]
|
||||
if c == nil {
|
||||
ct := models.UniqueConstraint
|
||||
if typ == "PRIMARY KEY" {
|
||||
ct = models.PrimaryKeyConstraint
|
||||
}
|
||||
if typ == "FOREIGN KEY" {
|
||||
ct = models.ForeignKeyConstraint
|
||||
}
|
||||
c = models.InitConstraint(name, ct)
|
||||
c.Schema = schema
|
||||
c.Table = table
|
||||
c.ReferencedSchema = rs
|
||||
c.ReferencedTable = rt
|
||||
out[name] = c
|
||||
}
|
||||
c.Columns = append(c.Columns, col)
|
||||
if rc != "" {
|
||||
c.ReferencedColumns = append(c.ReferencedColumns, rc)
|
||||
}
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
func (r *Reader) queryIndexes(schema, table string) (map[string]*models.Index, error) {
|
||||
rows, err := r.db.QueryContext(r.ctx, `SELECT INDEX_NAME, NON_UNIQUE, COLUMN_NAME, SEQ_IN_INDEX, INDEX_TYPE FROM information_schema.STATISTICS WHERE TABLE_SCHEMA=? AND TABLE_NAME=? AND INDEX_NAME <> 'PRIMARY' ORDER BY INDEX_NAME, SEQ_IN_INDEX`, schema, table)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
out := map[string]*models.Index{}
|
||||
for rows.Next() {
|
||||
var name, col, typ string
|
||||
var non, seq int
|
||||
if err := rows.Scan(&name, &non, &col, &seq, &typ); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
i := out[name]
|
||||
if i == nil {
|
||||
i = models.InitIndex(name, table, schema)
|
||||
i.Unique = non == 0
|
||||
i.Type = strings.ToLower(typ)
|
||||
out[name] = i
|
||||
}
|
||||
i.Columns = append(i.Columns, col)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
func (r *Reader) deriveRelationship(t *models.Table, c *models.Constraint) {
|
||||
n := fmt.Sprintf("%s_to_%s", t.Name, c.ReferencedTable)
|
||||
rel := models.InitRelationship(n, models.OneToMany)
|
||||
rel.FromTable = t.Name
|
||||
rel.FromSchema = t.Schema
|
||||
rel.FromColumns = append([]string(nil), c.Columns...)
|
||||
rel.ToTable = c.ReferencedTable
|
||||
rel.ToSchema = c.ReferencedSchema
|
||||
rel.ToColumns = append([]string(nil), c.ReferencedColumns...)
|
||||
rel.ForeignKey = c.Name
|
||||
t.Relationships[n] = rel
|
||||
}
|
||||
@@ -0,0 +1,22 @@
|
||||
package mysql
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"git.warky.dev/wdevs/relspecgo/pkg/readers"
|
||||
)
|
||||
|
||||
func TestReaderMapDataType(t *testing.T) {
|
||||
r := NewReader(&readers.ReaderOptions{})
|
||||
for _, tc := range []struct{ input, want string }{{"varchar(64)", "string"}, {"bigint unsigned", "int64"}, {"datetime", "timestamp"}, {"json", "json"}} {
|
||||
if got := r.mapDataType(tc.input); got != tc.want {
|
||||
t.Errorf("mapDataType(%q) = %q, want %q", tc.input, got, tc.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestReaderRequiresConnectionString(t *testing.T) {
|
||||
if _, err := NewReader(&readers.ReaderOptions{}).ReadDatabase(); err == nil {
|
||||
t.Fatal("expected missing connection string error")
|
||||
}
|
||||
}
|
||||
@@ -243,7 +243,11 @@ func (r *Reader) queryColumns(schemaName string) (map[string]map[string]*models.
|
||||
c.numeric_scale,
|
||||
c.udt_name,
|
||||
pg_catalog.format_type(a.atttypid, a.atttypmod) as formatted_data_type,
|
||||
col_description((c.table_schema||'.'||c.table_name)::regclass, c.ordinal_position) as description
|
||||
col_description((c.table_schema||'.'||c.table_name)::regclass, c.ordinal_position) as description,
|
||||
c.is_generated,
|
||||
c.generation_expression,
|
||||
c.is_identity,
|
||||
c.identity_generation
|
||||
FROM information_schema.columns c
|
||||
JOIN pg_catalog.pg_namespace n
|
||||
ON n.nspname = c.table_schema
|
||||
@@ -268,17 +272,36 @@ func (r *Reader) queryColumns(schemaName string) (map[string]map[string]*models.
|
||||
columnsMap := make(map[string]map[string]*models.Column)
|
||||
|
||||
for rows.Next() {
|
||||
var schema, tableName, columnName, isNullable, dataType, udtName, formattedDataType string
|
||||
var schema, tableName, columnName, isNullable, dataType, udtName, formattedDataType, isGenerated, isIdentity string
|
||||
var ordinalPosition int
|
||||
var columnDefault, description *string
|
||||
var columnDefault, description, generationExpression, identityGeneration *string
|
||||
var charMaxLength, numPrecision, numScale *int
|
||||
|
||||
if err := rows.Scan(&schema, &tableName, &columnName, &ordinalPosition, &columnDefault, &isNullable, &dataType, &charMaxLength, &numPrecision, &numScale, &udtName, &formattedDataType, &description); err != nil {
|
||||
if err := rows.Scan(&schema, &tableName, &columnName, &ordinalPosition, &columnDefault, &isNullable, &dataType, &charMaxLength, &numPrecision, &numScale, &udtName, &formattedDataType, &description, &isGenerated, &generationExpression, &isIdentity, &identityGeneration); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
column := models.InitColumn(columnName, tableName, schema)
|
||||
|
||||
// GENERATED ALWAYS ... STORED columns are computed by Postgres and cannot
|
||||
// have their default altered/dropped like a regular column.
|
||||
column.Generated = isGenerated == "ALWAYS"
|
||||
if generationExpression != nil {
|
||||
column.GenerationExpression = *generationExpression
|
||||
}
|
||||
|
||||
// GENERATED { ALWAYS | BY DEFAULT } AS IDENTITY columns are driven by an
|
||||
// internal sequence rather than a literal default (unlike serial columns,
|
||||
// they carry no pg_attrdef row at all), so they need the same DB-side
|
||||
// handling as generated columns even though the underlying mechanism differs.
|
||||
column.Identity = isIdentity == "YES"
|
||||
if identityGeneration != nil {
|
||||
column.IdentityGeneration = strings.ToUpper(strings.TrimSpace(*identityGeneration))
|
||||
}
|
||||
if column.Identity {
|
||||
column.AutoIncrement = true
|
||||
}
|
||||
|
||||
// Check if this is a serial type (has nextval default)
|
||||
hasNextval := false
|
||||
if columnDefault != nil {
|
||||
@@ -301,6 +324,24 @@ func (r *Reader) queryColumns(schemaName string) (map[string]map[string]*models.
|
||||
column.Description = *description
|
||||
}
|
||||
|
||||
if column.Generated {
|
||||
note := "GENERATED ALWAYS AS (" + column.GenerationExpression + ") STORED"
|
||||
if column.Description != "" {
|
||||
column.Description = column.Description + " " + note
|
||||
} else {
|
||||
column.Description = note
|
||||
}
|
||||
}
|
||||
|
||||
if column.Identity {
|
||||
note := "GENERATED " + column.IdentityGeneration + " AS IDENTITY"
|
||||
if column.Description != "" {
|
||||
column.Description = column.Description + " " + note
|
||||
} else {
|
||||
column.Description = note
|
||||
}
|
||||
}
|
||||
|
||||
if charMaxLength != nil {
|
||||
column.Length = *charMaxLength
|
||||
}
|
||||
@@ -381,31 +422,35 @@ func (r *Reader) queryPrimaryKeys(schemaName string) (map[string]*models.Constra
|
||||
// queryForeignKeys retrieves all foreign key constraints for a schema
|
||||
// Returns map[schema.table][]*Constraint
|
||||
func (r *Reader) queryForeignKeys(schemaName string) (map[string][]*models.Constraint, error) {
|
||||
// Columns are paired by position from pg_constraint so composite keys are not
|
||||
// cross-joined (information_schema.constraint_column_usage has no ordering).
|
||||
actionCase := func(col string) string {
|
||||
return `CASE ` + col + ` WHEN 'a' THEN 'NO ACTION' WHEN 'r' THEN 'RESTRICT' WHEN 'c' THEN 'CASCADE' WHEN 'n' THEN 'SET NULL' WHEN 'd' THEN 'SET DEFAULT' END`
|
||||
}
|
||||
query := `
|
||||
SELECT
|
||||
tc.table_schema,
|
||||
tc.table_name,
|
||||
tc.constraint_name,
|
||||
kcu.table_schema as foreign_table_schema,
|
||||
kcu.table_name as foreign_table_name,
|
||||
kcu.column_name as foreign_column,
|
||||
ccu.table_schema as referenced_table_schema,
|
||||
ccu.table_name as referenced_table_name,
|
||||
ccu.column_name as referenced_column,
|
||||
rc.update_rule,
|
||||
rc.delete_rule
|
||||
FROM information_schema.table_constraints tc
|
||||
JOIN information_schema.key_column_usage kcu
|
||||
ON tc.constraint_name = kcu.constraint_name
|
||||
AND tc.table_schema = kcu.table_schema
|
||||
JOIN information_schema.constraint_column_usage ccu
|
||||
ON ccu.constraint_name = tc.constraint_name
|
||||
JOIN information_schema.referential_constraints rc
|
||||
ON rc.constraint_name = tc.constraint_name
|
||||
AND rc.constraint_schema = tc.table_schema
|
||||
WHERE tc.constraint_type = 'FOREIGN KEY'
|
||||
AND tc.table_schema = $1
|
||||
ORDER BY tc.table_schema, tc.table_name, tc.constraint_name, kcu.ordinal_position
|
||||
ns.nspname AS table_schema,
|
||||
cl.relname AS table_name,
|
||||
con.conname AS constraint_name,
|
||||
ns.nspname AS foreign_table_schema,
|
||||
cl.relname AS foreign_table_name,
|
||||
att.attname AS foreign_column,
|
||||
fns.nspname AS referenced_table_schema,
|
||||
fcl.relname AS referenced_table_name,
|
||||
fatt.attname AS referenced_column,
|
||||
` + actionCase("con.confupdtype") + ` AS update_rule,
|
||||
` + actionCase("con.confdeltype") + ` AS delete_rule
|
||||
FROM pg_catalog.pg_constraint con
|
||||
JOIN pg_catalog.pg_class cl ON cl.oid = con.conrelid
|
||||
JOIN pg_catalog.pg_namespace ns ON ns.oid = cl.relnamespace
|
||||
JOIN pg_catalog.pg_class fcl ON fcl.oid = con.confrelid
|
||||
JOIN pg_catalog.pg_namespace fns ON fns.oid = fcl.relnamespace
|
||||
CROSS JOIN LATERAL unnest(con.conkey, con.confkey) WITH ORDINALITY AS k(attnum, fattnum, ord)
|
||||
JOIN pg_catalog.pg_attribute att ON att.attrelid = con.conrelid AND att.attnum = k.attnum
|
||||
JOIN pg_catalog.pg_attribute fatt ON fatt.attrelid = con.confrelid AND fatt.attnum = k.fattnum
|
||||
WHERE con.contype = 'f'
|
||||
AND ns.nspname = $1
|
||||
ORDER BY ns.nspname, cl.relname, con.conname, k.ord
|
||||
`
|
||||
|
||||
rows, err := r.conn.Query(r.ctx, query, schemaName)
|
||||
|
||||
@@ -34,11 +34,14 @@ func (r *Reader) ReadDatabase() (*models.Database, error) {
|
||||
return nil, fmt.Errorf("connection string is required")
|
||||
}
|
||||
|
||||
// Connect to the database
|
||||
// Connect to the database. This can take noticeable time across a slow network,
|
||||
// so report it before the driver starts the connection attempt.
|
||||
r.progress("Connecting to PostgreSQL...")
|
||||
if err := r.connect(); err != nil {
|
||||
return nil, fmt.Errorf("failed to connect: %w", err)
|
||||
}
|
||||
defer r.close()
|
||||
r.progress("Connected. Reading database metadata...")
|
||||
|
||||
// Get database name from connection
|
||||
var dbName string
|
||||
@@ -60,34 +63,42 @@ func (r *Reader) ReadDatabase() (*models.Database, error) {
|
||||
}
|
||||
|
||||
// Query all schemas
|
||||
r.progress("Discovering schemas...")
|
||||
schemas, err := r.querySchemas()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to query schemas: %w", err)
|
||||
}
|
||||
|
||||
// Process each schema
|
||||
for _, schema := range schemas {
|
||||
for schemaIndex, schema := range schemas {
|
||||
r.progress(fmt.Sprintf("Reading schema %q (%d/%d): tables...", schema.Name, schemaIndex+1, len(schemas)))
|
||||
// Query tables for this schema
|
||||
tables, err := r.queryTables(schema.Name)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to query tables for schema %s: %w", schema.Name, err)
|
||||
}
|
||||
schema.Tables = tables
|
||||
r.progress(fmt.Sprintf("Reading schema %q: found %d table(s).", schema.Name, len(tables)))
|
||||
|
||||
r.progress(fmt.Sprintf("Reading schema %q: views...", schema.Name))
|
||||
// Query views for this schema
|
||||
views, err := r.queryViews(schema.Name)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to query views for schema %s: %w", schema.Name, err)
|
||||
}
|
||||
schema.Views = views
|
||||
r.progress(fmt.Sprintf("Reading schema %q: found %d view(s).", schema.Name, len(views)))
|
||||
|
||||
r.progress(fmt.Sprintf("Reading schema %q: sequences...", schema.Name))
|
||||
// Query sequences for this schema
|
||||
sequences, err := r.querySequences(schema.Name)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to query sequences for schema %s: %w", schema.Name, err)
|
||||
}
|
||||
schema.Sequences = sequences
|
||||
r.progress(fmt.Sprintf("Reading schema %q: found %d sequence(s).", schema.Name, len(sequences)))
|
||||
|
||||
r.progress(fmt.Sprintf("Reading schema %q: extensions...", schema.Name))
|
||||
// Query extensions installed into this schema
|
||||
extensions, err := r.queryExtensions(schema.Name)
|
||||
if err != nil {
|
||||
@@ -99,12 +110,15 @@ func (r *Reader) ReadDatabase() (*models.Database, error) {
|
||||
}
|
||||
schema.Metadata["extensions"] = extensions
|
||||
}
|
||||
r.progress(fmt.Sprintf("Reading schema %q: found %d extension(s).", schema.Name, len(extensions)))
|
||||
|
||||
r.progress(fmt.Sprintf("Reading schema %q: columns...", schema.Name))
|
||||
// Query columns for tables and views
|
||||
columnsMap, err := r.queryColumns(schema.Name)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to query columns for schema %s: %w", schema.Name, err)
|
||||
}
|
||||
r.progress(fmt.Sprintf("Reading schema %q: found %d column(s).", schema.Name, countColumns(columnsMap)))
|
||||
|
||||
// Populate table columns
|
||||
for _, table := range schema.Tables {
|
||||
@@ -122,11 +136,13 @@ func (r *Reader) ReadDatabase() (*models.Database, error) {
|
||||
}
|
||||
}
|
||||
|
||||
r.progress(fmt.Sprintf("Reading schema %q: primary keys...", schema.Name))
|
||||
// Query primary keys
|
||||
primaryKeys, err := r.queryPrimaryKeys(schema.Name)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to query primary keys for schema %s: %w", schema.Name, err)
|
||||
}
|
||||
r.progress(fmt.Sprintf("Reading schema %q: found %d primary key(s).", schema.Name, len(primaryKeys)))
|
||||
|
||||
// Apply primary keys to tables
|
||||
for _, table := range schema.Tables {
|
||||
@@ -143,11 +159,13 @@ func (r *Reader) ReadDatabase() (*models.Database, error) {
|
||||
}
|
||||
}
|
||||
|
||||
r.progress(fmt.Sprintf("Reading schema %q: foreign keys...", schema.Name))
|
||||
// Query foreign keys
|
||||
foreignKeys, err := r.queryForeignKeys(schema.Name)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to query foreign keys for schema %s: %w", schema.Name, err)
|
||||
}
|
||||
r.progress(fmt.Sprintf("Reading schema %q: found %d foreign key(s).", schema.Name, countConstraints(foreignKeys)))
|
||||
|
||||
// Apply foreign keys to tables
|
||||
for _, table := range schema.Tables {
|
||||
@@ -161,11 +179,13 @@ func (r *Reader) ReadDatabase() (*models.Database, error) {
|
||||
}
|
||||
}
|
||||
|
||||
r.progress(fmt.Sprintf("Reading schema %q: unique constraints...", schema.Name))
|
||||
// Query unique constraints
|
||||
uniqueConstraints, err := r.queryUniqueConstraints(schema.Name)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to query unique constraints for schema %s: %w", schema.Name, err)
|
||||
}
|
||||
r.progress(fmt.Sprintf("Reading schema %q: found %d unique constraint(s).", schema.Name, countConstraints(uniqueConstraints)))
|
||||
|
||||
// Apply unique constraints to tables
|
||||
for _, table := range schema.Tables {
|
||||
@@ -177,11 +197,13 @@ func (r *Reader) ReadDatabase() (*models.Database, error) {
|
||||
}
|
||||
}
|
||||
|
||||
r.progress(fmt.Sprintf("Reading schema %q: check constraints...", schema.Name))
|
||||
// Query check constraints
|
||||
checkConstraints, err := r.queryCheckConstraints(schema.Name)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to query check constraints for schema %s: %w", schema.Name, err)
|
||||
}
|
||||
r.progress(fmt.Sprintf("Reading schema %q: found %d check constraint(s).", schema.Name, countConstraints(checkConstraints)))
|
||||
|
||||
// Apply check constraints to tables
|
||||
for _, table := range schema.Tables {
|
||||
@@ -193,11 +215,13 @@ func (r *Reader) ReadDatabase() (*models.Database, error) {
|
||||
}
|
||||
}
|
||||
|
||||
r.progress(fmt.Sprintf("Reading schema %q: indexes...", schema.Name))
|
||||
// Query indexes
|
||||
indexes, err := r.queryIndexes(schema.Name)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to query indexes for schema %s: %w", schema.Name, err)
|
||||
}
|
||||
r.progress(fmt.Sprintf("Reading schema %q: found %d index(es).", schema.Name, countIndexes(indexes)))
|
||||
|
||||
// Apply indexes to tables
|
||||
for _, table := range schema.Tables {
|
||||
@@ -226,10 +250,41 @@ func (r *Reader) ReadDatabase() (*models.Database, error) {
|
||||
// Add schema to database
|
||||
db.Schemas = append(db.Schemas, schema)
|
||||
}
|
||||
r.progress("PostgreSQL schema read complete.")
|
||||
|
||||
return db, nil
|
||||
}
|
||||
|
||||
func (r *Reader) progress(message string) {
|
||||
if r.options.Progress != nil {
|
||||
r.options.Progress(message)
|
||||
}
|
||||
}
|
||||
|
||||
func countColumns(columns map[string]map[string]*models.Column) int {
|
||||
total := 0
|
||||
for _, tableColumns := range columns {
|
||||
total += len(tableColumns)
|
||||
}
|
||||
return total
|
||||
}
|
||||
|
||||
func countConstraints(constraints map[string][]*models.Constraint) int {
|
||||
total := 0
|
||||
for _, tableConstraints := range constraints {
|
||||
total += len(tableConstraints)
|
||||
}
|
||||
return total
|
||||
}
|
||||
|
||||
func countIndexes(indexes map[string][]*models.Index) int {
|
||||
total := 0
|
||||
for _, tableIndexes := range indexes {
|
||||
total += len(tableIndexes)
|
||||
}
|
||||
return total
|
||||
}
|
||||
|
||||
// ReadSchema reads a single schema (returns the first schema from the database)
|
||||
func (r *Reader) ReadSchema() (*models.Schema, error) {
|
||||
db, err := r.ReadDatabase()
|
||||
|
||||
@@ -1,10 +1,14 @@
|
||||
package pgsql
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
|
||||
"git.warky.dev/wdevs/relspecgo/pkg/models"
|
||||
"git.warky.dev/wdevs/relspecgo/pkg/readers"
|
||||
)
|
||||
@@ -499,3 +503,60 @@ func TestMapDataType_ExtensionTypesPreserveModifiers(t *testing.T) {
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestReader_CompositeForeignKeyColumnsArePairedOnce(t *testing.T) {
|
||||
connStr := getTestConnectionString(t)
|
||||
|
||||
ctx := context.Background()
|
||||
conn, err := pgx.Connect(ctx, connStr)
|
||||
if err != nil {
|
||||
t.Fatalf("connect: %v", err)
|
||||
}
|
||||
defer conn.Close(ctx)
|
||||
|
||||
const schema = "relspec_fk_test"
|
||||
setup := []string{
|
||||
"DROP SCHEMA IF EXISTS " + schema + " CASCADE",
|
||||
"CREATE SCHEMA " + schema,
|
||||
"CREATE TABLE " + schema + ".parent (a int, b int, PRIMARY KEY (a, b))",
|
||||
"CREATE TABLE " + schema + ".child (x int, y int, CONSTRAINT fk_child_parent FOREIGN KEY (x, y) REFERENCES " + schema + ".parent (a, b) ON DELETE CASCADE)",
|
||||
}
|
||||
for _, stmt := range setup {
|
||||
if _, err := conn.Exec(ctx, stmt); err != nil {
|
||||
t.Fatalf("setup %q: %v", stmt, err)
|
||||
}
|
||||
}
|
||||
defer conn.Exec(ctx, "DROP SCHEMA IF EXISTS "+schema+" CASCADE")
|
||||
|
||||
reader := NewReader(&readers.ReaderOptions{ConnectionString: connStr})
|
||||
db, err := reader.ReadDatabase()
|
||||
if err != nil {
|
||||
t.Fatalf("ReadDatabase: %v", err)
|
||||
}
|
||||
|
||||
for _, s := range db.Schemas {
|
||||
if s.Name != schema {
|
||||
continue
|
||||
}
|
||||
for _, tbl := range s.Tables {
|
||||
if tbl.Name != "child" {
|
||||
continue
|
||||
}
|
||||
fk := tbl.Constraints["fk_child_parent"]
|
||||
if fk == nil {
|
||||
t.Fatal("foreign key fk_child_parent not read")
|
||||
}
|
||||
if got := strings.Join(fk.Columns, ","); got != "x,y" {
|
||||
t.Errorf("columns = %q, want x,y", got)
|
||||
}
|
||||
if got := strings.Join(fk.ReferencedColumns, ","); got != "a,b" {
|
||||
t.Errorf("referenced columns = %q, want a,b", got)
|
||||
}
|
||||
if fk.OnDelete != "CASCADE" || fk.OnUpdate != "NO ACTION" {
|
||||
t.Errorf("rules = %s/%s, want CASCADE/NO ACTION", fk.OnDelete, fk.OnUpdate)
|
||||
}
|
||||
return
|
||||
}
|
||||
}
|
||||
t.Fatal("test schema/table not found in read result")
|
||||
}
|
||||
|
||||
@@ -28,7 +28,7 @@ model User {
|
||||
id Int @id @default(autoincrement())
|
||||
}`
|
||||
|
||||
if err := os.WriteFile(schemaPath, []byte(content), 0644); err != nil {
|
||||
if err := os.WriteFile(schemaPath, []byte(content), 0o644); err != nil {
|
||||
t.Fatalf("failed to write schema: %v", err)
|
||||
}
|
||||
|
||||
@@ -58,7 +58,7 @@ model User {
|
||||
id Int @id @default(autoincrement())
|
||||
}`
|
||||
|
||||
if err := os.WriteFile(schemaPath, []byte(content), 0644); err != nil {
|
||||
if err := os.WriteFile(schemaPath, []byte(content), 0o644); err != nil {
|
||||
t.Fatalf("failed to write schema: %v", err)
|
||||
}
|
||||
|
||||
|
||||
@@ -28,6 +28,14 @@ type ReaderOptions struct {
|
||||
// Prisma7 enables Prisma 7-specific handling for Prisma schemas.
|
||||
Prisma7 bool
|
||||
|
||||
// StrictDirectives makes DBML dialect directives (@postgres:, @sqlite:, …)
|
||||
// fail on an unknown namespace or key instead of preserving them silently.
|
||||
StrictDirectives bool
|
||||
|
||||
// Progress receives human-readable status updates while a reader is working.
|
||||
// It is optional so library users can opt in without coupling readers to a UI.
|
||||
Progress func(string)
|
||||
|
||||
// Additional options can be added here as needed
|
||||
Metadata map[string]interface{}
|
||||
}
|
||||
|
||||
@@ -175,7 +175,6 @@ func (r *Reader) readScripts() ([]*models.Script, error) {
|
||||
|
||||
return nil
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
@@ -30,18 +30,18 @@ func TestReader_ReadDatabase(t *testing.T) {
|
||||
|
||||
for filename, content := range testFiles {
|
||||
filePath := filepath.Join(tempDir, filename)
|
||||
if err := os.WriteFile(filePath, []byte(content), 0644); err != nil {
|
||||
if err := os.WriteFile(filePath, []byte(content), 0o644); err != nil {
|
||||
t.Fatalf("Failed to create test file %s: %v", filename, err)
|
||||
}
|
||||
}
|
||||
|
||||
// Create subdirectory with additional script
|
||||
subDir := filepath.Join(tempDir, "migrations")
|
||||
if err := os.MkdirAll(subDir, 0755); err != nil {
|
||||
if err := os.MkdirAll(subDir, 0o755); err != nil {
|
||||
t.Fatalf("Failed to create subdirectory: %v", err)
|
||||
}
|
||||
subFile := filepath.Join(subDir, "3_001_add_column.sql")
|
||||
if err := os.WriteFile(subFile, []byte("ALTER TABLE users ADD COLUMN email TEXT;"), 0644); err != nil {
|
||||
if err := os.WriteFile(subFile, []byte("ALTER TABLE users ADD COLUMN email TEXT;"), 0o644); err != nil {
|
||||
t.Fatalf("Failed to create subdirectory file: %v", err)
|
||||
}
|
||||
|
||||
@@ -141,7 +141,7 @@ func TestReader_ReadSchema(t *testing.T) {
|
||||
|
||||
// Create test SQL file
|
||||
testFile := filepath.Join(tempDir, "1_001_test.sql")
|
||||
if err := os.WriteFile(testFile, []byte("SELECT 1;"), 0644); err != nil {
|
||||
if err := os.WriteFile(testFile, []byte("SELECT 1;"), 0o644); err != nil {
|
||||
t.Fatalf("Failed to create test file: %v", err)
|
||||
}
|
||||
|
||||
@@ -220,14 +220,14 @@ func TestReader_InvalidFilename(t *testing.T) {
|
||||
|
||||
for _, filename := range invalidFiles {
|
||||
filePath := filepath.Join(tempDir, filename)
|
||||
if err := os.WriteFile(filePath, []byte("SELECT 1;"), 0644); err != nil {
|
||||
if err := os.WriteFile(filePath, []byte("SELECT 1;"), 0o644); err != nil {
|
||||
t.Fatalf("Failed to create test file %s: %v", filename, err)
|
||||
}
|
||||
}
|
||||
|
||||
// Create one valid file
|
||||
validFile := filepath.Join(tempDir, "1_001_valid.sql")
|
||||
if err := os.WriteFile(validFile, []byte("SELECT 1;"), 0644); err != nil {
|
||||
if err := os.WriteFile(validFile, []byte("SELECT 1;"), 0o644); err != nil {
|
||||
t.Fatalf("Failed to create valid file: %v", err)
|
||||
}
|
||||
|
||||
@@ -277,7 +277,7 @@ func TestReader_HyphenFormat(t *testing.T) {
|
||||
|
||||
for filename, content := range testFiles {
|
||||
filePath := filepath.Join(tempDir, filename)
|
||||
if err := os.WriteFile(filePath, []byte(content), 0644); err != nil {
|
||||
if err := os.WriteFile(filePath, []byte(content), 0o644); err != nil {
|
||||
t.Fatalf("Failed to create test file %s: %v", filename, err)
|
||||
}
|
||||
}
|
||||
@@ -343,7 +343,7 @@ func TestReader_MixedFormat(t *testing.T) {
|
||||
|
||||
for filename, content := range testFiles {
|
||||
filePath := filepath.Join(tempDir, filename)
|
||||
if err := os.WriteFile(filePath, []byte(content), 0644); err != nil {
|
||||
if err := os.WriteFile(filePath, []byte(content), 0o644); err != nil {
|
||||
t.Fatalf("Failed to create test file %s: %v", filename, err)
|
||||
}
|
||||
}
|
||||
@@ -386,13 +386,13 @@ func TestReader_SkipSymlinks(t *testing.T) {
|
||||
|
||||
// Create a real SQL file
|
||||
realFile := filepath.Join(tempDir, "1_001_real_file.sql")
|
||||
if err := os.WriteFile(realFile, []byte("SELECT 1;"), 0644); err != nil {
|
||||
if err := os.WriteFile(realFile, []byte("SELECT 1;"), 0o644); err != nil {
|
||||
t.Fatalf("Failed to create real file: %v", err)
|
||||
}
|
||||
|
||||
// Create another file to link to
|
||||
targetFile := filepath.Join(tempDir, "2_001_target.sql")
|
||||
if err := os.WriteFile(targetFile, []byte("SELECT 2;"), 0644); err != nil {
|
||||
if err := os.WriteFile(targetFile, []byte("SELECT 2;"), 0o644); err != nil {
|
||||
t.Fatalf("Failed to create target file: %v", err)
|
||||
}
|
||||
|
||||
|
||||
@@ -40,6 +40,7 @@ options := &readers.ReaderOptions{
|
||||
- Uses pure Go driver (modernc.org/sqlite) - no CGo required
|
||||
- Supports both file path and connection string
|
||||
- Auto-increment detection for INTEGER PRIMARY KEY columns
|
||||
- Generated columns (virtual and stored) are read via `PRAGMA table_xinfo`; the expression is parsed from the `CREATE TABLE` SQL
|
||||
- Foreign keys require `PRAGMA foreign_keys = ON` to be set
|
||||
|
||||
## Example Schema
|
||||
|
||||
@@ -0,0 +1,116 @@
|
||||
package sqlite
|
||||
|
||||
import (
|
||||
"regexp"
|
||||
"strings"
|
||||
)
|
||||
|
||||
var generatedAsRegex = regexp.MustCompile(`(?is)\bAS\s*\(`)
|
||||
|
||||
// parseGeneratedExpression extracts the expression of a generated column from a CREATE
|
||||
// TABLE statement, e.g. `full TEXT GENERATED ALWAYS AS (a || b) STORED` yields `a || b`.
|
||||
// It returns "" when the column or its expression cannot be found.
|
||||
func parseGeneratedExpression(createSQL, columnName string) string {
|
||||
open := strings.Index(createSQL, "(")
|
||||
if open < 0 {
|
||||
return ""
|
||||
}
|
||||
|
||||
for _, def := range splitTopLevel(createSQL[open+1:]) {
|
||||
if !strings.EqualFold(firstIdentifier(def), columnName) {
|
||||
continue
|
||||
}
|
||||
loc := generatedAsRegex.FindStringIndex(def)
|
||||
if loc == nil {
|
||||
return ""
|
||||
}
|
||||
return balancedParens(def[loc[1]-1:])
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// splitTopLevel splits a table body on commas that are outside parentheses and quotes,
|
||||
// stopping at the parenthesis that closes the body.
|
||||
func splitTopLevel(body string) []string {
|
||||
var parts []string
|
||||
depth := 0
|
||||
var quote byte
|
||||
start := 0
|
||||
for i := 0; i < len(body); i++ {
|
||||
ch := body[i]
|
||||
if quote != 0 {
|
||||
if ch == quote {
|
||||
quote = 0
|
||||
}
|
||||
continue
|
||||
}
|
||||
switch ch {
|
||||
case '\'', '"', '`':
|
||||
quote = ch
|
||||
case '[':
|
||||
quote = ']'
|
||||
case '(':
|
||||
depth++
|
||||
case ')':
|
||||
if depth == 0 {
|
||||
return append(parts, body[start:i])
|
||||
}
|
||||
depth--
|
||||
case ',':
|
||||
if depth == 0 {
|
||||
parts = append(parts, body[start:i])
|
||||
start = i + 1
|
||||
}
|
||||
}
|
||||
}
|
||||
return append(parts, body[start:])
|
||||
}
|
||||
|
||||
// firstIdentifier returns the first (possibly quoted) identifier of a column definition.
|
||||
func firstIdentifier(def string) string {
|
||||
def = strings.TrimSpace(def)
|
||||
if def == "" {
|
||||
return ""
|
||||
}
|
||||
switch def[0] {
|
||||
case '"', '\'', '`':
|
||||
if end := strings.IndexByte(def[1:], def[0]); end >= 0 {
|
||||
return def[1 : 1+end]
|
||||
}
|
||||
case '[':
|
||||
if end := strings.IndexByte(def, ']'); end >= 0 {
|
||||
return def[1:end]
|
||||
}
|
||||
}
|
||||
if end := strings.IndexAny(def, " \t\r\n"); end >= 0 {
|
||||
return def[:end]
|
||||
}
|
||||
return def
|
||||
}
|
||||
|
||||
// balancedParens returns the text inside the parenthesis group that starts at s[0].
|
||||
func balancedParens(s string) string {
|
||||
depth := 0
|
||||
var quote byte
|
||||
for i := 0; i < len(s); i++ {
|
||||
ch := s[i]
|
||||
if quote != 0 {
|
||||
if ch == quote {
|
||||
quote = 0
|
||||
}
|
||||
continue
|
||||
}
|
||||
switch ch {
|
||||
case '\'', '"':
|
||||
quote = ch
|
||||
case '(':
|
||||
depth++
|
||||
case ')':
|
||||
depth--
|
||||
if depth == 0 {
|
||||
return strings.TrimSpace(s[1:i])
|
||||
}
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
@@ -0,0 +1,64 @@
|
||||
package sqlite
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"git.warky.dev/wdevs/relspecgo/pkg/readers"
|
||||
)
|
||||
|
||||
func TestParseGeneratedExpression(t *testing.T) {
|
||||
createSQL := `CREATE TABLE people (
|
||||
id INTEGER PRIMARY KEY,
|
||||
"first" TEXT,
|
||||
last TEXT,
|
||||
full_name TEXT GENERATED ALWAYS AS (coalesce("first", '') || ' ' || last) STORED,
|
||||
initials TEXT AS (substr("first", 1, 1) || substr(last, 1, 1)),
|
||||
plain TEXT NOT NULL
|
||||
)`
|
||||
|
||||
tests := []struct {
|
||||
column string
|
||||
want string
|
||||
}{
|
||||
{"full_name", `coalesce("first", '') || ' ' || last`},
|
||||
{"initials", `substr("first", 1, 1) || substr(last, 1, 1)`},
|
||||
{"plain", ""},
|
||||
{"missing", ""},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.column, func(t *testing.T) {
|
||||
assert.Equal(t, tt.want, parseGeneratedExpression(createSQL, tt.column))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestReader_GeneratedColumns(t *testing.T) {
|
||||
dbPath := filepath.Join(t.TempDir(), "gen.db")
|
||||
db, err := sql.Open("sqlite", dbPath)
|
||||
require.NoError(t, err)
|
||||
_, err = db.Exec(`CREATE TABLE people (
|
||||
id INTEGER PRIMARY KEY,
|
||||
first TEXT,
|
||||
last TEXT,
|
||||
full_name TEXT GENERATED ALWAYS AS (first || ' ' || last) STORED,
|
||||
initials TEXT AS (substr(first, 1, 1) || substr(last, 1, 1))
|
||||
)`)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, db.Close())
|
||||
|
||||
got, err := NewReader(&readers.ReaderOptions{FilePath: dbPath}).ReadDatabase()
|
||||
require.NoError(t, err)
|
||||
|
||||
cols := got.Schemas[0].Tables[0].Columns
|
||||
require.Contains(t, cols, "full_name", "generated columns must be read")
|
||||
assert.True(t, cols["full_name"].Generated)
|
||||
assert.Equal(t, "first || ' ' || last", cols["full_name"].GenerationExpression)
|
||||
assert.True(t, cols["initials"].Generated)
|
||||
assert.Equal(t, "substr(first, 1, 1) || substr(last, 1, 1)", cols["initials"].GenerationExpression)
|
||||
assert.False(t, cols["first"].Generated)
|
||||
}
|
||||
@@ -75,7 +75,8 @@ func (r *Reader) queryViews() ([]*models.View, error) {
|
||||
|
||||
// queryColumns retrieves all columns for a given table or view
|
||||
func (r *Reader) queryColumns(tableName string) (map[string]*models.Column, error) {
|
||||
query := fmt.Sprintf("PRAGMA table_info(%s)", tableName)
|
||||
// table_xinfo, unlike table_info, also lists generated columns (hidden = 2 virtual, 3 stored)
|
||||
query := fmt.Sprintf("PRAGMA table_xinfo(%s)", tableName)
|
||||
|
||||
rows, err := r.db.QueryContext(r.ctx, query)
|
||||
if err != nil {
|
||||
@@ -84,24 +85,38 @@ func (r *Reader) queryColumns(tableName string) (map[string]*models.Column, erro
|
||||
defer rows.Close()
|
||||
|
||||
columns := make(map[string]*models.Column)
|
||||
var tableSQL string
|
||||
tableSQLLoaded := false
|
||||
|
||||
for rows.Next() {
|
||||
var cid int
|
||||
var name, dataType string
|
||||
var notNull, pk int
|
||||
var notNull, pk, hidden int
|
||||
var defaultValue *string
|
||||
|
||||
if err := rows.Scan(&cid, &name, &dataType, ¬Null, &defaultValue, &pk); err != nil {
|
||||
if err := rows.Scan(&cid, &name, &dataType, ¬Null, &defaultValue, &pk, &hidden); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Hidden virtual-table columns (hidden = 1) are not part of the schema
|
||||
if hidden == 1 {
|
||||
continue
|
||||
}
|
||||
|
||||
column := models.InitColumn(name, tableName, "main")
|
||||
column.Type = r.mapDataType(strings.ToUpper(dataType))
|
||||
column.NotNull = (notNull == 1)
|
||||
column.IsPrimaryKey = (pk > 0)
|
||||
column.Sequence = uint(cid + 1)
|
||||
|
||||
if defaultValue != nil {
|
||||
if hidden == 2 || hidden == 3 {
|
||||
column.Generated = true
|
||||
if !tableSQLLoaded {
|
||||
tableSQL = r.tableSQL(tableName)
|
||||
tableSQLLoaded = true
|
||||
}
|
||||
column.GenerationExpression = parseGeneratedExpression(tableSQL, name)
|
||||
} else if defaultValue != nil {
|
||||
column.Default = *defaultValue
|
||||
}
|
||||
|
||||
@@ -116,6 +131,16 @@ func (r *Reader) queryColumns(tableName string) (map[string]*models.Column, erro
|
||||
return columns, rows.Err()
|
||||
}
|
||||
|
||||
// tableSQL returns the CREATE TABLE statement of a table, or "" when it cannot be read.
|
||||
func (r *Reader) tableSQL(tableName string) string {
|
||||
var sql string
|
||||
err := r.db.QueryRowContext(r.ctx, `SELECT sql FROM sqlite_master WHERE type = 'table' AND name = ?`, tableName).Scan(&sql)
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
return sql
|
||||
}
|
||||
|
||||
// isAutoIncrement checks if a column is autoincrement
|
||||
func (r *Reader) isAutoIncrement(tableName, columnName string) bool {
|
||||
// Check sqlite_sequence table or parse CREATE TABLE statement
|
||||
|
||||
@@ -114,6 +114,8 @@ export class Post {
|
||||
- `@JoinColumn()` - Foreign key column
|
||||
- `@Index()` - Index definition
|
||||
- `@Unique()` - Unique constraint
|
||||
- `asExpression` / `generatedType` - Generated column (`Generated` + `GenerationExpression`)
|
||||
- `@PrimaryGeneratedColumn('identity')`, `@Generated('identity')`, `generatedIdentity` - Identity column
|
||||
|
||||
## Notes
|
||||
|
||||
|
||||
@@ -128,7 +128,6 @@ func (r *Reader) extractEntities(content string) []entityInfo {
|
||||
scanner := bufio.NewScanner(strings.NewReader(content))
|
||||
|
||||
entityRegex := regexp.MustCompile(`^export\s+class\s+(\w+)`)
|
||||
decoratorRegex := regexp.MustCompile(`^\s*@(\w+)(\([^)]*\))?`)
|
||||
fieldRegex := regexp.MustCompile(`^\s*(\w+):\s*([^;]+);`)
|
||||
|
||||
var currentEntity *entityInfo
|
||||
@@ -145,8 +144,7 @@ func (r *Reader) extractEntities(content string) []entityInfo {
|
||||
}
|
||||
|
||||
// Check for decorator
|
||||
if matches := decoratorRegex.FindStringSubmatch(trimmed); matches != nil {
|
||||
decorator := matches[0]
|
||||
if decorator, ok := matchDecorator(trimmed); ok {
|
||||
pendingDecorators = append(pendingDecorators, decorator)
|
||||
continue
|
||||
}
|
||||
@@ -488,7 +486,11 @@ func (r *Reader) parseColumnDecorator(decorator string, column *models.Column, t
|
||||
column.IsPrimaryKey = true
|
||||
column.NotNull = true
|
||||
|
||||
if strings.Contains(decorator, "'uuid'") {
|
||||
if strings.Contains(decorator, "'identity'") {
|
||||
column.AutoIncrement = true
|
||||
column.Identity = true
|
||||
column.IdentityGeneration = parseGeneratedIdentity(decorator)
|
||||
} else if strings.Contains(decorator, "'uuid'") {
|
||||
column.Type = "uuid"
|
||||
column.Default = "gen_random_uuid()"
|
||||
} else if strings.Contains(decorator, "'increment'") || strings.Contains(decorator, "()") {
|
||||
@@ -497,6 +499,17 @@ func (r *Reader) parseColumnDecorator(decorator string, column *models.Column, t
|
||||
return
|
||||
}
|
||||
|
||||
// @Generated('identity') on a non-key column; generatedIdentity is read from @Column
|
||||
if strings.HasPrefix(decorator, "@Generated") {
|
||||
if strings.Contains(decorator, "'identity'") {
|
||||
column.Identity = true
|
||||
if column.IdentityGeneration == "" {
|
||||
column.IdentityGeneration = "BY DEFAULT"
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// @Column
|
||||
if strings.HasPrefix(decorator, "@Column") {
|
||||
r.parseColumnOptions(decorator, column, table)
|
||||
@@ -586,6 +599,14 @@ func (r *Reader) parseColumnOptions(decorator string, column *models.Column, tab
|
||||
}
|
||||
}
|
||||
|
||||
if matches := asExpressionRegex.FindStringSubmatch(content); matches != nil {
|
||||
column.Generated = true
|
||||
column.GenerationExpression = unescapeSingleQuoted(matches[1])
|
||||
}
|
||||
if strings.Contains(content, "generatedIdentity") {
|
||||
column.IdentityGeneration = parseGeneratedIdentity(content)
|
||||
}
|
||||
|
||||
if strings.Contains(content, "nullable: true") || strings.Contains(content, "nullable:true") {
|
||||
column.NotNull = false
|
||||
}
|
||||
@@ -834,3 +855,72 @@ func (r *Reader) getPrimaryKeyColumn(table *models.Table) *models.Column {
|
||||
|
||||
return pk
|
||||
}
|
||||
|
||||
var (
|
||||
asExpressionRegex = regexp.MustCompile(`asExpression:\s*'((?:\\.|[^\\'])*)'`)
|
||||
generatedIdentityRegexp = regexp.MustCompile(`generatedIdentity:\s*['"](ALWAYS|BY DEFAULT)['"]`)
|
||||
)
|
||||
|
||||
// parseGeneratedIdentity returns the identity generation mode named in a decorator
|
||||
// ("ALWAYS" or "BY DEFAULT"), defaulting to "BY DEFAULT" as TypeORM does.
|
||||
func parseGeneratedIdentity(decorator string) string {
|
||||
if matches := generatedIdentityRegexp.FindStringSubmatch(decorator); matches != nil {
|
||||
return matches[1]
|
||||
}
|
||||
return "BY DEFAULT"
|
||||
}
|
||||
|
||||
// unescapeSingleQuoted reverses escaping applied inside a quoted TypeScript string.
|
||||
func unescapeSingleQuoted(s string) string {
|
||||
var sb strings.Builder
|
||||
for i := 0; i < len(s); i++ {
|
||||
if s[i] == '\\' && i+1 < len(s) {
|
||||
i++
|
||||
}
|
||||
sb.WriteByte(s[i])
|
||||
}
|
||||
return sb.String()
|
||||
}
|
||||
|
||||
var decoratorNameRegex = regexp.MustCompile(`^@\w+`)
|
||||
|
||||
// matchDecorator returns the decorator at the start of line, including its argument list.
|
||||
// Parentheses inside quoted strings (e.g. a generated column expression) do not end it.
|
||||
func matchDecorator(line string) (string, bool) {
|
||||
name := decoratorNameRegex.FindString(line)
|
||||
if name == "" {
|
||||
return "", false
|
||||
}
|
||||
rest := line[len(name):]
|
||||
if !strings.HasPrefix(rest, "(") {
|
||||
return name, true
|
||||
}
|
||||
|
||||
depth := 0
|
||||
var quote byte
|
||||
for i := 0; i < len(rest); i++ {
|
||||
ch := rest[i]
|
||||
if quote != 0 {
|
||||
switch ch {
|
||||
case '\\':
|
||||
i++
|
||||
case quote:
|
||||
quote = 0
|
||||
}
|
||||
continue
|
||||
}
|
||||
switch ch {
|
||||
case '\'', '"', '`':
|
||||
quote = ch
|
||||
case '(':
|
||||
depth++
|
||||
case ')':
|
||||
depth--
|
||||
if depth == 0 {
|
||||
return name + rest[:i+1], true
|
||||
}
|
||||
}
|
||||
}
|
||||
// Unterminated argument list: keep the whole line, as a best effort
|
||||
return line, true
|
||||
}
|
||||
|
||||
@@ -192,7 +192,7 @@ func mapKeyLess(a, b reflect.Value) bool {
|
||||
|
||||
// MapGet safely gets a value from a map by key
|
||||
// Returns nil if key doesn't exist or not a map
|
||||
func MapGet(m interface{}, key interface{}) interface{} {
|
||||
func MapGet(m, key interface{}) interface{} {
|
||||
v := reflect.ValueOf(m)
|
||||
v, ok := Deref(v)
|
||||
if !ok {
|
||||
|
||||
+13
-12
@@ -111,6 +111,7 @@ func (n *SqlNull[T]) Scan(value any) error {
|
||||
return n.FromString(fmt.Sprintf("%v", value))
|
||||
}
|
||||
}
|
||||
|
||||
func (n *SqlNull[T]) FromString(s string) error {
|
||||
s = strings.TrimSpace(s)
|
||||
n.Valid = false
|
||||
@@ -444,55 +445,55 @@ type (
|
||||
SqlUUID = SqlNull[uuid.UUID]
|
||||
)
|
||||
|
||||
// SqlTimeStamp - Timestamp with custom formatting (YYYY-MM-DDTHH:MM:SS).
|
||||
// SqlTimeStamp - Timestamp serialized as RFC3339.
|
||||
type SqlTimeStamp struct{ SqlNull[time.Time] }
|
||||
|
||||
func (t SqlTimeStamp) MarshalJSON() ([]byte, error) {
|
||||
if !t.Valid || t.Val.IsZero() || t.Val.Before(time.Date(0002, 1, 1, 0, 0, 0, 0, time.UTC)) {
|
||||
if !t.Valid || t.Val.IsZero() || t.Val.Before(time.Date(0o002, 1, 1, 0, 0, 0, 0, time.UTC)) {
|
||||
return []byte("null"), nil
|
||||
}
|
||||
return fmt.Appendf(nil, `"%s"`, t.Val.Format("2006-01-02T15:04:05")), nil
|
||||
return fmt.Appendf(nil, `"%s"`, t.Val.Format(time.RFC3339)), nil
|
||||
}
|
||||
|
||||
func (t *SqlTimeStamp) UnmarshalJSON(b []byte) error {
|
||||
if err := t.SqlNull.UnmarshalJSON(b); err != nil {
|
||||
return err
|
||||
}
|
||||
if t.Valid && (t.Val.IsZero() || t.Val.Format("2006-01-02T15:04:05") == "0001-01-01T00:00:00") {
|
||||
if t.Valid && (t.Val.IsZero() || t.Val.Format(time.RFC3339) == "0001-01-01T00:00:00Z") {
|
||||
t.Valid = false
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (t SqlTimeStamp) Value() (driver.Value, error) {
|
||||
if !t.Valid || t.Val.IsZero() || t.Val.Before(time.Date(0002, 1, 1, 0, 0, 0, 0, time.UTC)) {
|
||||
if !t.Valid || t.Val.IsZero() || t.Val.Before(time.Date(0o002, 1, 1, 0, 0, 0, 0, time.UTC)) {
|
||||
return nil, nil
|
||||
}
|
||||
return t.Val.Format("2006-01-02T15:04:05"), nil
|
||||
return t.Val.Format(time.RFC3339), nil
|
||||
}
|
||||
|
||||
func (t SqlTimeStamp) MarshalYAML() (any, error) {
|
||||
if !t.Valid || t.Val.IsZero() || t.Val.Before(time.Date(0002, 1, 1, 0, 0, 0, 0, time.UTC)) {
|
||||
if !t.Valid || t.Val.IsZero() || t.Val.Before(time.Date(0o002, 1, 1, 0, 0, 0, 0, time.UTC)) {
|
||||
return nil, nil
|
||||
}
|
||||
return t.Val.Format("2006-01-02T15:04:05"), nil
|
||||
return t.Val.Format(time.RFC3339), nil
|
||||
}
|
||||
|
||||
func (t *SqlTimeStamp) UnmarshalYAML(value *yaml.Node) error {
|
||||
if err := t.SqlNull.UnmarshalYAML(value); err != nil {
|
||||
return err
|
||||
}
|
||||
if t.Valid && (t.Val.IsZero() || t.Val.Format("2006-01-02T15:04:05") == "0001-01-01T00:00:00") {
|
||||
if t.Valid && (t.Val.IsZero() || t.Val.Format(time.RFC3339) == "0001-01-01T00:00:00Z") {
|
||||
t.Valid = false
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (t SqlTimeStamp) MarshalXML(e *xml.Encoder, start xml.StartElement) error {
|
||||
if !t.Valid || t.Val.IsZero() || t.Val.Before(time.Date(0002, 1, 1, 0, 0, 0, 0, time.UTC)) {
|
||||
if !t.Valid || t.Val.IsZero() || t.Val.Before(time.Date(0o002, 1, 1, 0, 0, 0, 0, time.UTC)) {
|
||||
return e.EncodeElement("", start)
|
||||
}
|
||||
return e.EncodeElement(t.Val.Format("2006-01-02T15:04:05"), start)
|
||||
return e.EncodeElement(t.Val.Format(time.RFC3339), start)
|
||||
}
|
||||
|
||||
func (t *SqlTimeStamp) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error {
|
||||
@@ -510,7 +511,7 @@ func (t *SqlTimeStamp) UnmarshalXML(d *xml.Decoder, start xml.StartElement) erro
|
||||
return err
|
||||
}
|
||||
t.Val = tm
|
||||
t.Valid = !tm.IsZero() && tm.Format("2006-01-02T15:04:05") != "0001-01-01T00:00:00"
|
||||
t.Valid = !tm.IsZero() && tm.Format(time.RFC3339) != "0001-01-01T00:00:00Z"
|
||||
return nil
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,157 @@
|
||||
package sqltypes
|
||||
|
||||
import (
|
||||
"database/sql/driver"
|
||||
"encoding/json"
|
||||
"math"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestSqlNull_ValueScalarCases(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
input SqlNull[any]
|
||||
want driver.Value
|
||||
}{
|
||||
{name: "invalid", input: SqlNull[any]{}, want: nil},
|
||||
{name: "integer", input: Null[any](int64(42), true), want: int64(42)},
|
||||
{name: "string", input: Null[any]("hello", true), want: "hello"},
|
||||
{name: "boolean", input: Null[any](true, true), want: true},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got, err := tt.input.Value()
|
||||
if err != nil {
|
||||
t.Fatalf("Value returned error: %v", err)
|
||||
}
|
||||
if got != tt.want {
|
||||
t.Errorf("Value() = %v (%T), want %v (%T)", got, got, tt.want, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestSqlNull_Int64Conversions(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
input SqlNull[any]
|
||||
want int64
|
||||
}{
|
||||
{name: "invalid", input: SqlNull[any]{}, want: 0},
|
||||
{name: "signed integer", input: Null[any](int32(-12), true), want: -12},
|
||||
{name: "unsigned integer", input: Null[any](uint16(12), true), want: 12},
|
||||
{name: "float truncates", input: Null[any](float64(12.9), true), want: 12},
|
||||
{name: "numeric string", input: Null[any]("123", true), want: 123},
|
||||
{name: "invalid string", input: Null[any]("not a number", true), want: 0},
|
||||
{name: "true", input: Null[any](true, true), want: 1},
|
||||
{name: "false", input: Null[any](false, true), want: 0},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
if got := tt.input.Int64(); got != tt.want {
|
||||
t.Errorf("Int64() = %d, want %d", got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestSqlNull_Float64Conversions(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
input SqlNull[any]
|
||||
want float64
|
||||
}{
|
||||
{name: "invalid", input: SqlNull[any]{}, want: 0},
|
||||
{name: "float", input: Null[any](float32(1.25), true), want: 1.25},
|
||||
{name: "signed integer", input: Null[any](int64(-12), true), want: -12},
|
||||
{name: "unsigned integer", input: Null[any](uint16(12), true), want: 12},
|
||||
{name: "numeric string", input: Null[any]("12.5", true), want: 12.5},
|
||||
{name: "invalid string", input: Null[any]("not a number", true), want: 0},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
if got := tt.input.Float64(); got != tt.want {
|
||||
t.Errorf("Float64() = %v, want %v", got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestSqlDate_JSONNullAndInvalid(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
json string
|
||||
valid bool
|
||||
}{
|
||||
{name: "null", json: "null", valid: false},
|
||||
{name: "invalid date", json: `"not-a-date"`, valid: false},
|
||||
{name: "valid date", json: `"2024-01-15"`, valid: true},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
var got SqlDate
|
||||
if err := json.Unmarshal([]byte(tt.json), &got); err != nil {
|
||||
t.Fatalf("UnmarshalJSON returned error: %v", err)
|
||||
}
|
||||
if got.Valid != tt.valid {
|
||||
t.Errorf("Valid = %v, want %v", got.Valid, tt.valid)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
if data, err := json.Marshal(SqlDate{}); err != nil {
|
||||
t.Fatalf("MarshalJSON returned error: %v", err)
|
||||
} else if string(data) != "null" {
|
||||
t.Errorf("MarshalJSON() = %s, want null", data)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSqlTypeNowConstructors(t *testing.T) {
|
||||
before := time.Now()
|
||||
timestamp := SqlTimeStampNow()
|
||||
date := SqlDateNow()
|
||||
tm := SqlTimeNow()
|
||||
after := time.Now()
|
||||
|
||||
for name, got := range map[string]time.Time{
|
||||
"timestamp": timestamp.Time(),
|
||||
"date": date.Time(),
|
||||
"time": tm.Time(),
|
||||
} {
|
||||
if !got.After(before) && !got.Equal(before) || got.After(after) {
|
||||
t.Errorf("%s constructor returned %v outside [%v, %v]", name, got, before, after)
|
||||
}
|
||||
}
|
||||
if !timestamp.Valid || !date.Valid || !tm.Valid {
|
||||
t.Fatal("Now constructors must return valid values")
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewSqlAndToJSONDT(t *testing.T) {
|
||||
if got := NewSql[int64]("42"); !got.Valid || got.Val != 42 {
|
||||
t.Errorf("NewSql[int64](\"42\") = %#v, want valid 42", got)
|
||||
}
|
||||
if got := NewSql[int64](nil); got.Valid {
|
||||
t.Errorf("NewSql[int64](nil) = %#v, want invalid", got)
|
||||
}
|
||||
if got := NewSqlFloat32(1.5); !got.Valid || got.Val != 1.5 {
|
||||
t.Errorf("NewSqlFloat32(1.5) = %#v, want valid 1.5", got)
|
||||
}
|
||||
|
||||
when := time.Date(2024, 1, 15, 10, 30, 45, 0, time.UTC)
|
||||
if got := ToJSONDT(when); got != "2024-01-15T10:30:45Z" {
|
||||
t.Errorf("ToJSONDT() = %q, want RFC3339 timestamp", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSqlNull_Float64PreservesInfinity(t *testing.T) {
|
||||
got := Null[float64](math.Inf(1), true).Float64()
|
||||
if !math.IsInf(got, 1) {
|
||||
t.Errorf("Float64() = %v, want +Inf", got)
|
||||
}
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user