Compare commits

...
8 Commits
Author SHA1 Message Date
warkanum 32cf25ff73 chore(release): update package version to 1.0.91
Release / test (push) Successful in 54s
Release / release (push) Successful in 2m31s
Release / AUR package (relspec) (push) Successful in 50s
Release / pkg-rpm (push) Successful in 1m50s
Release / AUR package (relspec-bin) (push) Successful in 52s
Release / pkg-deb (push) Successful in 1m32s
2026-10-12 22:15:53 +02:00
warkanum 032093e8c0 feat(packaging): add relspec-bin AUR package
- Add linux/arch/relspec-bin/PKGBUILD using prebuilt release tarballs
- Move source PKGBUILD to linux/arch/relspec/PKGBUILD
- Release: build static binaries, pack linux tarballs (binary + LICENSE)
- AUR job: matrix over relspec and relspec-bin
- Makefile: bump pkgver in both PKGBUILDs
2026-10-12 22:15:07 +02:00
warkanum 69a27784d1 chore(release): update package version to 1.0.90
Release / test (push) Successful in 54s
Release / release (push) Successful in 3m12s
Release / pkg-aur (push) Successful in 52s
Release / pkg-deb (push) Successful in 1m22s
Release / pkg-rpm (push) Successful in 1m55s
2026-10-12 22:06:59 +02:00
warkanum 1d93255575 fix(assets): improve error reporting for asset failures
fix(embed): update embedDirectives to return result directly

fix(executor): clean up import order and formatting

fix(syntax_test): change file permission syntax to octal

fix(mysql): update file permission syntax to octal

fix(pgsql): update file permission syntax to octal

fix(sqlex): simplify Walk function parameter handling

fix(transform): adjust identifier comparison logic

fix(template): clean up function formatting and error handling
2026-10-12 22:04:40 +02:00
warkanum 05466a92b1 fix: preserve MySQL schema fidelity 2026-10-12 21:56:48 +02:00
warkanum dd97413b8e fix: contain generated files and assets and harden SQL rendering 2026-10-12 20:59:46 +02:00
warkanum 2fa48335d3 fix: address audit execution failures and unsafe type conversions 2026-10-12 19:53:19 +02:00
Hein bfe9852826 chore(Makefile): update govulncheck command to use pinned toolchain 2026-10-12 09:49:25 +02:00
104 changed files with 5908 additions and 1392 deletions
+35 -8
View File
@@ -71,11 +71,18 @@ jobs:
EXT="" EXT=""
[ "$GOOS" = "windows" ] && EXT=".exe" [ "$GOOS" = "windows" ] && EXT=".exe"
NAME="relspec-${GOOS}-${GOARCH}${EXT}" NAME="relspec-${GOOS}-${GOARCH}${EXT}"
GOOS="$GOOS" GOARCH="$GOARCH" go build \ GOOS="$GOOS" GOARCH="$GOARCH" CGO_ENABLED=0 go build \
-trimpath \ -trimpath \
-ldflags "$LDFLAGS" \ -ldflags "$LDFLAGS" \
-o "$NAME" ./cmd/relspec -o "$NAME" ./cmd/relspec
echo "Built $NAME" echo "Built $NAME"
if [ "$GOOS" = "linux" ]; then
TARNAME="relspec_${VERSION#v}_${GOOS}_${GOARCH}.tar.gz"
cp "$NAME" relspec
tar czf "$TARNAME" relspec LICENSE
rm relspec
echo "Packed $TARNAME"
fi
done done
- name: Build Windows installer - name: Build Windows installer
@@ -121,7 +128,7 @@ jobs:
exit 1 exit 1
fi fi
for f in relspec-*; do for f in relspec-* relspec_*.tar.gz; do
echo "Uploading $f..." echo "Uploading $f..."
curl -s -X POST "${UPLOAD_URL}?name=${f}" \ curl -s -X POST "${UPLOAD_URL}?name=${f}" \
-H "Authorization: token ${GITHUB_TOKEN}" \ -H "Authorization: token ${GITHUB_TOKEN}" \
@@ -132,8 +139,13 @@ jobs:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
pkg-aur: pkg-aur:
name: AUR package (${{ matrix.pkg }})
needs: release needs: release
runs-on: ubuntu-latest runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
pkg: [relspec, relspec-bin]
steps: steps:
- uses: actions/checkout@v4 - uses: actions/checkout@v4
@@ -145,6 +157,7 @@ jobs:
VERSION="${{ github.event.inputs.tag || github.ref_name }}" VERSION="${{ github.event.inputs.tag || github.ref_name }}"
PKGVER="${VERSION#v}" PKGVER="${VERSION#v}"
PKG="${{ matrix.pkg }}"
AUR_KEY_PATH="$HOME/.ssh/aur" AUR_KEY_PATH="$HOME/.ssh/aur"
AUR_KNOWN_HOSTS="$HOME/.ssh/known_hosts" AUR_KNOWN_HOSTS="$HOME/.ssh/known_hosts"
@@ -183,10 +196,15 @@ jobs:
# Clone AUR repo # Clone AUR repo
GIT_SSH_COMMAND="ssh -o IdentitiesOnly=yes -o StrictHostKeyChecking=yes -o UserKnownHostsFile=$AUR_KNOWN_HOSTS -i $AUR_KEY_PATH" \ GIT_SSH_COMMAND="ssh -o IdentitiesOnly=yes -o StrictHostKeyChecking=yes -o UserKnownHostsFile=$AUR_KNOWN_HOSTS -i $AUR_KEY_PATH" \
git clone ssh://aur@aur.archlinux.org/relspec.git aur-repo git clone "ssh://aur@aur.archlinux.org/${PKG}.git" aur-repo
if [ -f aur-repo/PKGBUILD ]; then
CURRENT_PKGVER=$(awk -F= '/^pkgver=/ {print $2; exit}' aur-repo/PKGBUILD | tr -d "[:space:]") CURRENT_PKGVER=$(awk -F= '/^pkgver=/ {print $2; exit}' aur-repo/PKGBUILD | tr -d "[:space:]")
CURRENT_PKGREL=$(awk -F= '/^pkgrel=/ {print $2; exit}' aur-repo/PKGBUILD | tr -d "[:space:]") CURRENT_PKGREL=$(awk -F= '/^pkgrel=/ {print $2; exit}' aur-repo/PKGBUILD | tr -d "[:space:]")
else
CURRENT_PKGVER=""
CURRENT_PKGREL=0
fi
if [ "$CURRENT_PKGVER" = "$PKGVER" ]; then if [ "$CURRENT_PKGVER" = "$PKGVER" ]; then
case "$CURRENT_PKGREL" in case "$CURRENT_PKGREL" in
@@ -202,16 +220,25 @@ jobs:
PKGREL=1 PKGREL=1
fi fi
echo "Publishing AUR package version ${PKGVER}-${PKGREL}" echo "Publishing AUR package ${PKG} ${PKGVER}-${PKGREL}"
# Compute SHA256 of the source archive from the same URL the PKGBUILD will download. # Compute SHA256 of the artifacts from the same URLs the PKGBUILD will download.
SHA=$(curl -fsSL "https://git.warky.dev/wdevs/relspecgo/releases/download/v${PKGVER}/relspec-v${PKGVER}-source.zip" | sha256sum | cut -d' ' -f1) DL="https://git.warky.dev/wdevs/relspecgo/releases/download/v${PKGVER}"
if [ "$PKG" = "relspec-bin" ]; then
SHA_AMD64=$(curl -fsSL "${DL}/relspec_${PKGVER}_linux_amd64.tar.gz" | sha256sum | cut -d' ' -f1)
SHA_ARM64=$(curl -fsSL "${DL}/relspec_${PKGVER}_linux_arm64.tar.gz" | sha256sum | cut -d' ' -f1)
SUMS=(-e "s/^sha256sums_x86_64=.*/sha256sums_x86_64=('${SHA_AMD64}')/" \
-e "s/^sha256sums_aarch64=.*/sha256sums_aarch64=('${SHA_ARM64}')/")
else
SHA=$(curl -fsSL "${DL}/relspec-v${PKGVER}-source.zip" | sha256sum | cut -d' ' -f1)
SUMS=(-e "s/^sha256sums=.*/sha256sums=('${SHA}')/")
fi
# Update PKGBUILD — keep remote source URL, bump version/checksum, and increment pkgrel for same-version rebuilds. # Update PKGBUILD — keep remote source URL, bump version/checksum, and increment pkgrel for same-version rebuilds.
sed -e "s/^pkgver=.*/pkgver=${PKGVER}/" \ sed -e "s/^pkgver=.*/pkgver=${PKGVER}/" \
-e "s/^pkgrel=.*/pkgrel=${PKGREL}/" \ -e "s/^pkgrel=.*/pkgrel=${PKGREL}/" \
-e "s/^sha256sums=.*/sha256sums=('${SHA}')/" \ "${SUMS[@]}" \
linux/arch/PKGBUILD > aur-repo/PKGBUILD "linux/arch/${PKG}/PKGBUILD" > aur-repo/PKGBUILD
# Generate .SRCINFO inside an Arch container (docker cp avoids DinD volume mount issues) # Generate .SRCINFO inside an Arch container (docker cp avoids DinD volume mount issues)
CID=$(docker run -d archlinux:latest sleep infinity) CID=$(docker run -d archlinux:latest sleep infinity)
+9 -8
View File
@@ -17,7 +17,8 @@ GOCLEAN=$(GOCMD) clean
# Tool versions (compiled on demand via `go run` so they match the local toolchain) # Tool versions (compiled on demand via `go run` so they match the local toolchain)
GOLANGCI_LINT = go run github.com/golangci/golangci-lint/v2/cmd/golangci-lint@latest GOLANGCI_LINT = go run github.com/golangci/golangci-lint/v2/cmd/golangci-lint@latest
STATICCHECK = build/tools/staticcheck STATICCHECK = build/tools/staticcheck
GOVULNCHECK = go run golang.org/x/vuln/cmd/govulncheck@latest # Scan with the toolchain pinned in go.mod (what CI uses), so stdlib vulns match CI.
GOVULNCHECK = GOTOOLCHAIN=go$(shell awk '/^go /{print $$2; exit}' go.mod) go run golang.org/x/vuln/cmd/govulncheck@latest
# Version information # Version information
# VERSION file is the single source of truth (bumped by `make release-version`). # VERSION file is the single source of truth (bumped by `make release-version`).
@@ -218,7 +219,7 @@ docker-test-integration: docker-up ## Start DB and run integration tests
$(GOTEST) -v ./pkg/readers/pgsql/ -count=1 || (make docker-down && exit 1) $(GOTEST) -v ./pkg/readers/pgsql/ -count=1 || (make docker-down && exit 1)
@make docker-down @make docker-down
release: lint fmt-check test build ## Run lint, format check, tests, build, then create and push a new release tag release: lint fmt-check govulncheck test build ## Run lint, format check, govulncheck, tests, build, then create and push a new release tag
@echo "Creating new release..." @echo "Creating new release..."
@latest_tag=$$(git describe --tags --abbrev=0 2>/dev/null || echo ""); \ @latest_tag=$$(git describe --tags --abbrev=0 2>/dev/null || echo ""); \
if [ -z "$$latest_tag" ]; then \ if [ -z "$$latest_tag" ]; then \
@@ -237,9 +238,9 @@ release: lint fmt-check test build ## Run lint, format check, tests, build, then
PKGVER=$${version#v}; \ PKGVER=$${version#v}; \
echo "Updating package metadata to $$PKGVER..."; \ echo "Updating package metadata to $$PKGVER..."; \
echo "$$PKGVER" > VERSION; \ echo "$$PKGVER" > VERSION; \
sed -i "s/^pkgver=.*/pkgver=$$PKGVER/" linux/arch/PKGBUILD; \ sed -i "s/^pkgver=.*/pkgver=$$PKGVER/" linux/arch/relspec/PKGBUILD linux/arch/relspec-bin/PKGBUILD; \
sed -i "s/^Version:.*/Version: $$PKGVER/" linux/centos/relspec.spec; \ sed -i "s/^Version:.*/Version: $$PKGVER/" linux/centos/relspec.spec; \
git add VERSION linux/arch/PKGBUILD linux/centos/relspec.spec; \ git add VERSION linux/arch/relspec/PKGBUILD linux/arch/relspec-bin/PKGBUILD linux/centos/relspec.spec; \
if ! git diff --cached --quiet; then \ if ! git diff --cached --quiet; then \
git commit -m "chore(release): update package version to $$PKGVER"; \ git commit -m "chore(release): update package version to $$PKGVER"; \
fi; \ fi; \
@@ -252,7 +253,7 @@ release: lint fmt-check test build ## Run lint, format check, tests, build, then
git push origin "$$version"; \ git push origin "$$version"; \
echo "Tag $$version created and pushed to remote repository." echo "Tag $$version created and pushed to remote repository."
release-version: lint fmt-check staticcheck ## Run lint, format check and staticcheck, then auto-increment patch version, update VERSION/package files, commit, tag, and push release-version: lint fmt-check staticcheck govulncheck ## Run lint, format check, staticcheck and govulncheck, then auto-increment patch version, update VERSION/package files, commit, tag, and push
@CURRENT=$$([ -f VERSION ] && cat VERSION | tr -d '[:space:]' || (git describe --tags --abbrev=0 2>/dev/null | sed 's/^v//') || echo "0.0.0"); \ @CURRENT=$$([ -f VERSION ] && cat VERSION | tr -d '[:space:]' || (git describe --tags --abbrev=0 2>/dev/null | sed 's/^v//') || echo "0.0.0"); \
MAJOR=$$(echo $$CURRENT | cut -d. -f1); \ MAJOR=$$(echo $$CURRENT | cut -d. -f1); \
MINOR=$$(echo $$CURRENT | cut -d. -f2); \ MINOR=$$(echo $$CURRENT | cut -d. -f2); \
@@ -261,15 +262,15 @@ release-version: lint fmt-check staticcheck ## Run lint, format check and static
NEXT="v$$PKGVER"; \ NEXT="v$$PKGVER"; \
echo "Current: v$$CURRENT → Next: $$NEXT"; \ echo "Current: v$$CURRENT → Next: $$NEXT"; \
echo "$$PKGVER" > VERSION; \ echo "$$PKGVER" > VERSION; \
sed -i "s/^pkgver=.*/pkgver=$$PKGVER/" linux/arch/PKGBUILD; \ sed -i "s/^pkgver=.*/pkgver=$$PKGVER/" linux/arch/relspec/PKGBUILD linux/arch/relspec-bin/PKGBUILD; \
sed -i "s/^Version:.*/Version: $$PKGVER/" linux/centos/relspec.spec; \ sed -i "s/^Version:.*/Version: $$PKGVER/" linux/centos/relspec.spec; \
git add VERSION linux/arch/PKGBUILD linux/centos/relspec.spec; \ git add VERSION linux/arch/relspec/PKGBUILD linux/arch/relspec-bin/PKGBUILD linux/centos/relspec.spec; \
git commit -m "chore(release): update package version to $$PKGVER"; \ git commit -m "chore(release): update package version to $$PKGVER"; \
git tag -a "$$NEXT" -m "Release $$NEXT"; \ git tag -a "$$NEXT" -m "Release $$NEXT"; \
git push origin HEAD "$$NEXT"; \ git push origin HEAD "$$NEXT"; \
echo "Pushed $$NEXT — release workflow triggered" echo "Pushed $$NEXT — release workflow triggered"
rerelease: lint fmt-check ## Move the latest tag to HEAD and force push it rerelease: lint fmt-check govulncheck ## Move the latest tag to HEAD and force push it
@TAG=$$(git describe --tags --abbrev=0 2>/dev/null); \ @TAG=$$(git describe --tags --abbrev=0 2>/dev/null); \
if [ -z "$$TAG" ]; then echo "No existing tags found"; exit 1; fi; \ if [ -z "$$TAG" ]; then echo "No existing tags found"; exit 1; fi; \
echo "Moving $$TAG to $$(git rev-parse --short HEAD)"; \ echo "Moving $$TAG to $$(git rev-parse --short HEAD)"; \
+76
View File
@@ -41,6 +41,9 @@ relspec convert --from pgsql --from-conn "postgres://user:pass@localhost/mydb" \
# DBML → PostgreSQL DDL # DBML → PostgreSQL DDL
relspec convert --from dbml --from-path schema.dbml --to pgsql --to-path schema.sql relspec convert --from dbml --from-path schema.dbml --to pgsql --to-path schema.sql
# Lowercase model names and structured references before output
relspec convert --from dbml --from-path schema.dbml --to pgsql --to-path schema.sql --lowercase
# PostgreSQL → SQLite (auto flattens schemas) # PostgreSQL → SQLite (auto flattens schemas)
relspec convert --from pgsql --from-conn "postgres://..." --to sqlite --to-path schema.sql relspec convert --from pgsql --from-conn "postgres://..." --to sqlite --to-path schema.sql
@@ -57,6 +60,44 @@ connections or `--dry-run`). Source files are polled every `--watch-interval`
is ignored so generating into the source tree does not loop. Conversion errors is ignored so generating into the source tree does not loop. Conversion errors
are printed and watching continues. are printed and watching continues.
#### Identifier casing and PostgreSQL quoting
PostgreSQL output always double-quotes model identifiers, including reserved
words, and preserves their case by default. For example, schema `Sales` and table
`Order` produce `"Sales"."Order"`; with `--lowercase`, they produce
`"sales"."order"`. Quoted names are case-sensitive in PostgreSQL.
`--lowercase` is available on `convert`, `batch`, `merge`, `split`, `templ`, and
`job run`. It lowercases model names and structured references on a copy and
rejects collisions such as `Users` and `users` in the same schema. SQL expressions,
defaults, comments, enum values, and data retain their case. Adjust identifier
references inside your own SQL expressions when opting into lowercasing.
Generated PostgreSQL SQL also escapes identifier quotes and literal values and
chooses dollar-quote delimiters that do not collide with function or procedural
block contents. Raw SQL expressions remain executable input. See the
[PostgreSQL writer documentation](pkg/writers/pgsql/README.md#identifier-and-value-handling)
for library options and details.
#### MySQL fidelity and strict conversion
MySQL output now preserves foreign keys and their actions, checks, indexes,
unsigned types, dimensions, generated columns, and native defaults. Known
unsupported features produce warnings. Add `--strict-conversion` to reject
lossy MySQL output before writing a file or executing DDL:
```bash
relspec convert --from mysql --from-conn "$MYSQL_DSN" \
--to mysql --to-path schema.sql --strict-conversion
```
Schema jobs can set `options.strict_conversion: true`. This check currently
covers MySQL output; it does not guarantee lossless conversion to every format.
The live suite covers MySQL 8.4 / InnoDB, including JSON/YAML intermediates.
See the [MySQL capability matrix](docs/MYSQL_FIDELITY.md) for unsupported objects,
metadata, SQL-mode requirements, and execution limits. Output creates tables in
existing databases; it is not a data copy or a rerunnable migration.
### `batch` — Convert many inputs in one run ### `batch` — Convert many inputs in one run
Converts each input independently (one output per input, unlike `--from-list` Converts each input independently (one output per input, unlike `--from-list`
@@ -165,6 +206,7 @@ jobs:
overwrite: true overwrite: true
options: options:
flatten_schema: false flatten_schema: false
lowercase_identifiers: true # opt in; defaults to false
logfile: .relspec/log/build-schema.log logfile: .relspec/log/build-schema.log
``` ```
@@ -177,6 +219,40 @@ any job runs. Path-like fields also support `${NAME}` environment-variable
references, which are expanded and safety-checked during pre-flight. See references, which are expanded and safety-checked during pre-flight. See
[docs/JOB_FILES.md](docs/JOB_FILES.md). [docs/JOB_FILES.md](docs/JOB_FILES.md).
Use `options.lowercase_identifiers: true` per job, or
`relspec job run build-schema --lowercase` to enable lowercasing for schema outputs
throughout the selected job plan. For MySQL schema output,
`options.strict_conversion: true` or `--strict-conversion` rejects known lossy
conversions before writing.
### `scripts` — SQL scripts and embedded assets
```bash
relspec scripts list --dir ./migrations
relspec scripts execute --dir ./migrations --conn "$DATABASE_URL"
```
Execution stops on the first error by default. `--continue-on-error` (alias
`--coe`) attempts later scripts and still exits nonzero if any script fails.
Scripts can embed nearby files using a standalone directive comment:
```sql
-- @embed: path=assets/message.txt var=:message mode=text
INSERT INTO messages (body) VALUES (:message);
```
Embed replacement preserves dollar text such as `$1`, `${2}`, and `$HOME` exactly.
Placeholders are replaced only in executable SQL; quoted strings, quoted
identifiers, comments, casts, and dollar-quoted bodies retain their contents.
Inserted content is never scanned for further placeholders. Asset-call named
parameters follow the same SQL lexical boundaries. See
[docs/SCRIPTS_COMMAND.md](docs/SCRIPTS_COMMAND.md) for file containment, limits,
binary embedding, and execution details.
MySQL DDL execution keeps each generated statement whole, preserving multiline
defaults containing semicolons or comment-like text.
### `edit` — Interactive TUI editor ### `edit` — Interactive TUI editor
```bash ```bash
+1 -1
View File
@@ -1 +1 @@
1.0.89 1.0.91
+2 -1
View File
@@ -203,7 +203,8 @@ func runAssetsExecute(cmd *cobra.Command, args []string) error {
if len(failures) > 0 { if len(failures) > 0 {
fmt.Fprintf(os.Stderr, "Failed: %d\n", len(failures)) fmt.Fprintf(os.Stderr, "Failed: %d\n", len(failures))
fmt.Fprintf(os.Stderr, "\n⚠ Failed Entries Summary (%d failed):\n", len(failures)) fmt.Fprintf(os.Stderr, "\n⚠ Failed Entries Summary (%d failed):\n", len(failures))
for i, f := range failures { for i := range failures {
f := &failures[i]
fmt.Fprintf(os.Stderr, " %d. %s (Priority=%d, Sequence=%d)\n Error: %v\n", fmt.Fprintf(os.Stderr, " %d. %s (Priority=%d, Sequence=%d)\n Error: %v\n",
i+1, filepath.Base(f.item.Entry.File), f.item.Priority, f.item.Sequence, f.err) i+1, filepath.Base(f.item.Entry.File), f.item.Priority, f.item.Sequence, f.err)
} }
+1
View File
@@ -68,6 +68,7 @@ func init() {
batchCmd.Flags().StringVar(&batchNullableTypes, "types", "", "Nullable type package for code-gen writers (bun/gorm)") batchCmd.Flags().StringVar(&batchNullableTypes, "types", "", "Nullable type package for code-gen writers (bun/gorm)")
batchCmd.Flags().StringVar(&batchNullableArrays, "array-nullable", "", "Nullable array representation for the Bun writer") batchCmd.Flags().StringVar(&batchNullableArrays, "array-nullable", "", "Nullable array representation for the Bun writer")
batchCmd.Flags().BoolVar(&batchContinueOnError, "continue-on-error", false, "Prepend \\set ON_ERROR_STOP off to generated SQL (pgsql output only)") batchCmd.Flags().BoolVar(&batchContinueOnError, "continue-on-error", false, "Prepend \\set ON_ERROR_STOP off to generated SQL (pgsql output only)")
batchCmd.Flags().BoolVar(&batchContinueOnError, "coe", false, "Alias for --continue-on-error")
batchCmd.Flags().BoolVar(&batchKeepGoing, "keep-going", false, "Process remaining inputs after a failure; still exits non-zero") batchCmd.Flags().BoolVar(&batchKeepGoing, "keep-going", false, "Process remaining inputs after a failure; still exits non-zero")
batchCmd.Flags().BoolVar(&batchDryRun, "dry-run", false, "Read and validate every input and print the plan without writing any output") batchCmd.Flags().BoolVar(&batchDryRun, "dry-run", false, "Read and validate every input and print the plan without writing any output")
+31 -4
View File
@@ -28,6 +28,7 @@ import (
"git.warky.dev/wdevs/relspecgo/pkg/readers/sqlite" "git.warky.dev/wdevs/relspecgo/pkg/readers/sqlite"
"git.warky.dev/wdevs/relspecgo/pkg/readers/typeorm" "git.warky.dev/wdevs/relspecgo/pkg/readers/typeorm"
"git.warky.dev/wdevs/relspecgo/pkg/readers/yaml" "git.warky.dev/wdevs/relspecgo/pkg/readers/yaml"
"git.warky.dev/wdevs/relspecgo/pkg/transform"
"git.warky.dev/wdevs/relspecgo/pkg/writers" "git.warky.dev/wdevs/relspecgo/pkg/writers"
wbun "git.warky.dev/wdevs/relspecgo/pkg/writers/bun" wbun "git.warky.dev/wdevs/relspecgo/pkg/writers/bun"
wdbml "git.warky.dev/wdevs/relspecgo/pkg/writers/dbml" wdbml "git.warky.dev/wdevs/relspecgo/pkg/writers/dbml"
@@ -180,12 +181,12 @@ Examples:
} }
func init() { func init() {
convertCmd.Flags().StringVar(&convertSourceType, "from", "", "Source format (dbml, dctx, drawdb, graphql, json, yaml, gorm, bun, drizzle, prisma, typeorm, pgsql, sqlite)") convertCmd.Flags().StringVar(&convertSourceType, "from", "", "Source format (dbml, dctx, drawdb, graphql, json, yaml, gorm, bun, drizzle, prisma, typeorm, pgsql, mssql, mysql, sqlite)")
convertCmd.Flags().StringVar(&convertSourcePath, "from-path", "", "Source file path (for file-based formats)") convertCmd.Flags().StringVar(&convertSourcePath, "from-path", "", "Source file path (for file-based formats)")
convertCmd.Flags().StringVar(&convertSourceConn, "from-conn", "", "Source connection string (for pgsql) or file path (for sqlite)") convertCmd.Flags().StringVar(&convertSourceConn, "from-conn", "", "Source connection string (pgsql, mssql, mysql) or file path (sqlite)")
convertCmd.Flags().StringSliceVar(&convertFromList, "from-list", nil, "Comma-separated list of source file paths to read and merge (mutually exclusive with --from-path)") convertCmd.Flags().StringSliceVar(&convertFromList, "from-list", nil, "Comma-separated list of source file paths to read and merge (mutually exclusive with --from-path)")
convertCmd.Flags().StringVar(&convertTargetType, "to", "", "Target format (dbml, dctx, drawdb, graphql, json, yaml, gorm, bun, drizzle, prisma, typeorm, pgsql)") convertCmd.Flags().StringVar(&convertTargetType, "to", "", "Target format (dbml, dctx, drawdb, graphql, json, yaml, gorm, bun, drizzle, prisma, typeorm, pgsql, mssql, mysql, sqlite)")
convertCmd.Flags().StringVar(&convertTargetPath, "to-path", "", "Target output path (file or directory)") convertCmd.Flags().StringVar(&convertTargetPath, "to-path", "", "Target output path (file or directory)")
convertCmd.Flags().StringVar(&convertPackageName, "package", "", "Package name (for code generation formats like gorm/bun)") convertCmd.Flags().StringVar(&convertPackageName, "package", "", "Package name (for code generation formats like gorm/bun)")
convertCmd.Flags().StringVar(&convertSchemaFilter, "schema", "", "Filter to a specific schema by name (required for formats like dctx that only support single schemas)") convertCmd.Flags().StringVar(&convertSchemaFilter, "schema", "", "Filter to a specific schema by name (required for formats like dctx that only support single schemas)")
@@ -193,6 +194,7 @@ func init() {
convertCmd.Flags().StringVar(&convertNullableTypes, "types", "", "Nullable type package for code-gen writers (bun/gorm): 'baselib' (default, Go pointer types), 'stdlib' (database/sql), or 'sqltypes'") convertCmd.Flags().StringVar(&convertNullableTypes, "types", "", "Nullable type package for code-gen writers (bun/gorm): 'baselib' (default, Go pointer types), 'stdlib' (database/sql), or 'sqltypes'")
convertCmd.Flags().StringVar(&convertNullableArrays, "array-nullable", "", "Nullable PostgreSQL array representation for the Bun writer in stdlib/baselib --types mode: 'slice' (default, plain slice) or 'pointer_slice' (*[]T, distinguishes NULL from '{}')") convertCmd.Flags().StringVar(&convertNullableArrays, "array-nullable", "", "Nullable PostgreSQL array representation for the Bun writer in stdlib/baselib --types mode: 'slice' (default, plain slice) or 'pointer_slice' (*[]T, distinguishes NULL from '{}')")
convertCmd.Flags().BoolVar(&convertContinueOnError, "continue-on-error", false, "Prepend \\set ON_ERROR_STOP off to generated SQL so psql continues past errors (pgsql output only)") convertCmd.Flags().BoolVar(&convertContinueOnError, "continue-on-error", false, "Prepend \\set ON_ERROR_STOP off to generated SQL so psql continues past errors (pgsql output only)")
convertCmd.Flags().BoolVar(&convertContinueOnError, "coe", false, "Alias for --continue-on-error")
convertCmd.Flags().StringVar(&convertExtraFields, "extra-fields", "", "Path to JSON file containing extra Bun model fields to inject (bun output only); fields support target_table, name, type, bun_tag, json_tag, comment") convertCmd.Flags().StringVar(&convertExtraFields, "extra-fields", "", "Path to JSON file containing extra Bun model fields to inject (bun output only); fields support target_table, name, type, bun_tag, json_tag, comment")
convertCmd.Flags().BoolVar(&convertDryRun, "dry-run", false, "Read and validate the input and print the plan without writing any output") convertCmd.Flags().BoolVar(&convertDryRun, "dry-run", false, "Read and validate the input and print the plan without writing any output")
@@ -447,10 +449,22 @@ func readDatabaseForConvert(dbType, filePath, connString string) (*models.Databa
return db, nil return db, nil
} }
func writeDatabase(db *models.Database, dbType, outputPath, packageName, schemaFilter string, flattenSchema bool, nullableTypes, nullableArrays string, continueOnError bool, extraFields string) error { func writeDatabase(db *models.Database, dbType, outputPath, packageName, schemaFilter string, flattenSchema bool, nullableTypes, nullableArrays string, continueOnError bool, extraFields string, strict ...bool) error {
if lowercaseIdentifiers {
var err error
schemaFilter = strings.ToLower(schemaFilter)
db, err = transform.LowercaseIdentifiers(db)
if err != nil {
return err
}
}
var writer writers.Writer var writer writers.Writer
writerOpts := newWriterOptions(outputPath, packageName, flattenSchema, nullableTypes, nullableArrays, continueOnError) writerOpts := newWriterOptions(outputPath, packageName, flattenSchema, nullableTypes, nullableArrays, continueOnError)
if len(strict) > 0 {
writerOpts.StrictConversion = writerOpts.StrictConversion || strict[0]
}
if extraFields != "" { if extraFields != "" {
extraFieldsJSON, err := loadExtraFields(dbType, extraFields) extraFieldsJSON, err := loadExtraFields(dbType, extraFields)
if err != nil { if err != nil {
@@ -600,6 +614,19 @@ func validateWriteTarget(db *models.Database, dbType, packageName, schemaFilter,
"pgsql", "postgres", "postgresql", "sql", "pgsql", "postgres", "postgresql", "sql",
"mssql", "sqlserver", "mssql2016", "mssql2017", "mssql2019", "mssql2022", "mssql", "sqlserver", "mssql2016", "mssql2017", "mssql2019", "mssql2022",
"sqlite", "sqlite3", "prisma", "typeorm", "graphql", "gql": "sqlite", "sqlite3", "prisma", "typeorm", "graphql", "gql":
case "mysql", "mariadb":
selected := db
if schemaFilter != "" {
selected = &models.Database{}
for _, s := range db.Schemas {
if s.Name == schemaFilter {
selected.Schemas = append(selected.Schemas, s)
}
}
}
if _, err := wmysql.NewWriter(newWriterOptions("", packageName, false, "", "", false)).Statements(selected); err != nil {
return err
}
case "gorm", "bun": case "gorm", "bun":
if packageName == "" { if packageName == "" {
return fmt.Errorf("package name is required for %s format (use --package flag)", dbType) return fmt.Errorf("package name is required for %s format (use --package flag)", dbType)
+22 -2
View File
@@ -19,6 +19,7 @@ import (
"git.warky.dev/wdevs/relspecgo/pkg/models" "git.warky.dev/wdevs/relspecgo/pkg/models"
"git.warky.dev/wdevs/relspecgo/pkg/readers" "git.warky.dev/wdevs/relspecgo/pkg/readers"
"git.warky.dev/wdevs/relspecgo/pkg/readers/sqldir" "git.warky.dev/wdevs/relspecgo/pkg/readers/sqldir"
"git.warky.dev/wdevs/relspecgo/pkg/transform"
"git.warky.dev/wdevs/relspecgo/pkg/writers" "git.warky.dev/wdevs/relspecgo/pkg/writers"
wpgsql "git.warky.dev/wdevs/relspecgo/pkg/writers/pgsql" wpgsql "git.warky.dev/wdevs/relspecgo/pkg/writers/pgsql"
"git.warky.dev/wdevs/relspecgo/pkg/writers/sqlexec" "git.warky.dev/wdevs/relspecgo/pkg/writers/sqlexec"
@@ -522,6 +523,12 @@ func runTemplJob(rj *resolvedJob, lg *jobLogger) error {
return fmt.Errorf("schema not found: %s", schema) return fmt.Errorf("schema not found: %s", schema)
} }
} }
if lowercaseIdentifiers || rj.job.Options.LowercaseIdentifiers {
db, err = transform.LowercaseIdentifiers(db)
if err != nil {
return err
}
}
mode := rj.job.Mode mode := rj.job.Mode
if mode == "" { if mode == "" {
mode = "database" mode = "database"
@@ -777,6 +784,8 @@ func runScriptsExecJob(rj *resolvedJob, lg *jobLogger) error {
lg.logf("executing %d script(s) against database env:%s", len(combined.Scripts), rj.outputConnEnv) lg.logf("executing %d script(s) against database env:%s", len(combined.Scripts), rj.outputConnEnv)
writer := sqlexec.NewWriter(&writers.WriterOptions{ writer := sqlexec.NewWriter(&writers.WriterOptions{
ContinueOnError: rj.job.Options.ContinueOnError,
AtomicExecution: rj.job.Options.AtomicExecution,
Metadata: map[string]any{ Metadata: map[string]any{
"connection_string": rj.outputConn, "connection_string": rj.outputConn,
"ignore_errors": rj.job.Options.ContinueOnError, "ignore_errors": rj.job.Options.ContinueOnError,
@@ -791,7 +800,7 @@ func runScriptsExecJob(rj *resolvedJob, lg *jobLogger) error {
success, _ := opts.Metadata["execution_success"].(int) success, _ := opts.Metadata["execution_success"].(int)
failed, _ := opts.Metadata["execution_failed"].(int) failed, _ := opts.Metadata["execution_failed"].(int)
lg.logf("executed %d script(s): %d succeeded, %d failed", total, success, failed) lg.logf("executed %d script(s): %d succeeded, %d failed", total, success, failed)
if failed > 0 && !rj.job.Options.ContinueOnError { if failed > 0 {
return fmt.Errorf("%d script(s) failed", failed) return fmt.Errorf("%d script(s) failed", failed)
} }
return nil return nil
@@ -835,7 +844,17 @@ func inputLabel(ri resolvedInput) string {
// writeJobOutput writes db to the job's output target (file or database). // writeJobOutput writes db to the job's output target (file or database).
func writeJobOutput(rj *resolvedJob, db *models.Database, lg *jobLogger) error { func writeJobOutput(rj *resolvedJob, db *models.Database, lg *jobLogger) error {
if lowercaseIdentifiers || rj.job.Options.LowercaseIdentifiers {
var err error
db, err = transform.LowercaseIdentifiers(db)
if err != nil {
return err
}
}
o := rj.job.Options o := rj.job.Options
if lowercaseIdentifiers || o.LowercaseIdentifiers {
o.Schema = strings.ToLower(o.Schema)
}
format := strings.ToLower(rj.job.Output.Format) format := strings.ToLower(rj.job.Output.Format)
if rj.outputConn != "" { if rj.outputConn != "" {
@@ -844,6 +863,7 @@ func writeJobOutput(rj *resolvedJob, db *models.Database, lg *jobLogger) error {
} }
lg.logf("writing output to database env:%s", rj.outputConnEnv) lg.logf("writing output to database env:%s", rj.outputConnEnv)
writerOpts := newWriterOptions("", o.Package, o.FlattenSchema, o.Types, o.ArrayNullable, o.ContinueOnError) writerOpts := newWriterOptions("", o.Package, o.FlattenSchema, o.Types, o.ArrayNullable, o.ContinueOnError)
writerOpts.AtomicExecution = o.AtomicExecution
writerOpts.Metadata = map[string]interface{}{"connection_string": rj.outputConn, "full_ddl": o.FullDDL} writerOpts.Metadata = map[string]interface{}{"connection_string": rj.outputConn, "full_ddl": o.FullDDL}
return wpgsql.NewWriter(writerOpts).WriteDatabase(db) return wpgsql.NewWriter(writerOpts).WriteDatabase(db)
} }
@@ -854,7 +874,7 @@ func writeJobOutput(rj *resolvedJob, db *models.Database, lg *jobLogger) error {
lg.logf("writing output: %s (%s)", rj.outputPath, format) lg.logf("writing output: %s (%s)", rj.outputPath, format)
write := func(target string) error { write := func(target string) error {
return writeDatabase(db, format, target, o.Package, o.Schema, o.FlattenSchema, o.Types, o.ArrayNullable, o.ContinueOnError, "") return writeDatabase(db, format, target, o.Package, o.Schema, o.FlattenSchema, o.Types, o.ArrayNullable, o.ContinueOnError, "", o.StrictConversion)
} }
// Single-file formats are written to a temp file and renamed into place so // Single-file formats are written to a temp file and renamed into place so
// a failure never leaves a partial or truncated output. Directory-emitting // a failure never leaves a partial or truncated output. Directory-emitting
+105
View File
@@ -2,11 +2,15 @@ package main
import ( import (
"bytes" "bytes"
"context"
"fmt"
"os" "os"
"path/filepath" "path/filepath"
"strings" "strings"
"testing" "testing"
"time"
"github.com/jackc/pgx/v5"
"github.com/spf13/cobra" "github.com/spf13/cobra"
"git.warky.dev/wdevs/relspecgo/pkg/jobs" "git.warky.dev/wdevs/relspecgo/pkg/jobs"
@@ -741,3 +745,104 @@ jobs:
t.Fatalf("expected templ format rejection, got %v", err) t.Fatalf("expected templ format rejection, got %v", err)
} }
} }
func TestLive_JobPostgresFailureStopsDependents(t *testing.T) {
connString := os.Getenv("RELSPEC_TEST_PG_CONN")
if connString == "" {
t.Skip("RELSPEC_TEST_PG_CONN not set")
}
t.Setenv("RELSPEC_TEST_JOB_OUTPUT_CONN", connString)
for _, continueOnError := range []string{"false", "true"} {
t.Run("continue="+continueOnError, func(t *testing.T) {
manifest := `version: 1
jobs:
apply:
command: convert
inputs:
- path: schema/core.dbml
format: dbml
output:
format: pgsql
conn_env: RELSPEC_TEST_JOB_OUTPUT_CONN
options:
full_ddl: true
continue_on_error: ` + continueOnError + `
logfile: .relspec/apply.log
dependent:
command: convert
depends_on: [apply]
inputs:
- path: schema/core.dbml
format: dbml
output:
format: json
path: build/dependent.json
logfile: .relspec/dependent.log
`
dir := jobFixture(t, manifest)
schema := fmt.Sprintf("job_audit_%d", time.Now().UnixNano())
conn, err := pgx.Connect(context.Background(), connString)
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() {
_, _ = conn.Exec(context.Background(), "DROP SCHEMA IF EXISTS "+schema+" CASCADE")
_ = conn.Close(context.Background())
})
// An invalid column type prevents table creation in this isolated schema.
writeFile(t, filepath.Join(dir, "schema", "core.dbml"), "Table "+schema+".users {\n id no_such_type_audit_xyz\n}\n")
set := mustLoadSet(t, filepath.Join(dir, "relspec.yml"))
err = executeJobPlan(set, "dependent", false, false, &bytes.Buffer{})
if err == nil || !strings.Contains(err.Error(), `job "apply" failed`) {
t.Fatalf("expected database failure propagation, got %v", err)
}
for _, path := range []string{"build/dependent.json", ".relspec/dependent.log"} {
if _, err := os.Stat(filepath.Join(dir, path)); !os.IsNotExist(err) {
t.Fatalf("dependent must not start: path=%s err=%v", path, err)
}
}
logData, err := os.ReadFile(filepath.Join(dir, ".relspec/apply.log"))
if err != nil || !strings.Contains(string(logData), "FAILED") {
t.Fatalf("failed application must be logged: %s err=%v", logData, err)
}
})
}
}
func TestLive_JobScriptContinuationStillStopsDependents(t *testing.T) {
connection := os.Getenv("RELSPEC_TEST_PG_CONN")
if connection == "" {
t.Skip("RELSPEC_TEST_PG_CONN not set")
}
t.Setenv("RELSPEC_TEST_EXEC_CONN", connection)
dir := jobFixture(t, `version: 1
jobs:
migrate:
command: scripts-exec
script_dirs: [migrations]
output:
conn_env: RELSPEC_TEST_EXEC_CONN
options:
continue_on_error: true
logfile: .relspec/migrate.log
dependent:
command: convert
depends_on: [migrate]
inputs:
- path: schema/core.dbml
format: dbml
output:
format: json
path: build/dependent.json
`)
writeFile(t, filepath.Join(dir, "migrations", "1_001_bad.sql"), "SELECT 1 / 0;\n")
writeFile(t, filepath.Join(dir, "migrations", "1_002_good.sql"), "SELECT 1;\n")
set := mustLoadSet(t, filepath.Join(dir, "relspec.yml"))
err := executeJobPlan(set, "dependent", false, false, &bytes.Buffer{})
if err == nil || !strings.Contains(err.Error(), `job "migrate" failed`) {
t.Fatalf("failed scripts must fail job: %v", err)
}
if _, err := os.Stat(filepath.Join(dir, "build", "dependent.json")); !os.IsNotExist(err) {
t.Fatalf("dependent ran: %v", err)
}
}
+82
View File
@@ -0,0 +1,82 @@
package main
import (
"bytes"
"os"
"path/filepath"
"strings"
"testing"
"git.warky.dev/wdevs/relspecgo/pkg/models"
)
func TestLowercaseFlagAndFileOutput(t *testing.T) {
prior := lowercaseIdentifiers
t.Cleanup(func() { lowercaseIdentifiers = prior })
for _, cmd := range []struct {
name string
flag bool
}{{"convert", convertCmd.Flags().Lookup("lowercase") != nil}, {"batch", batchCmd.Flags().Lookup("lowercase") != nil}, {"merge", mergeCmd.Flags().Lookup("lowercase") != nil}, {"templ", templCmd.Flags().Lookup("lowercase") != nil}, {"job", jobCmd.PersistentFlags().Lookup("lowercase") != nil}} {
if !cmd.flag {
t.Fatalf("missing %s --lowercase", cmd.name)
}
}
if err := convertCmd.Flags().Set("lowercase", "true"); err != nil {
t.Fatal(err)
}
path := filepath.Join(t.TempDir(), "schema.sql")
db := models.InitDatabase("DB")
s := models.InitSchema("App")
table := models.InitTable("Users", "App")
col := models.InitColumn("Name", "Users", "App")
col.Type = "text"
col.Default = "Keep CASE"
table.Columns[col.Name] = col
s.Tables = append(s.Tables, table)
db.Schemas = append(db.Schemas, s)
if err := writeDatabase(db, "pgsql", path, "", "", false, "", "", false, ""); err != nil {
t.Fatal(err)
}
data, err := os.ReadFile(path)
if err != nil {
t.Fatal(err)
}
if !strings.Contains(string(data), `"app"."users"`) || !strings.Contains(string(data), `"name" text DEFAULT 'Keep CASE'`) {
t.Fatalf("%s", data)
}
if table.Name != "Users" {
t.Fatal("input mutated")
}
}
func TestJobLowercaseOption(t *testing.T) {
prior := lowercaseIdentifiers
lowercaseIdentifiers = false
t.Cleanup(func() { lowercaseIdentifiers = prior })
dir := t.TempDir()
writeTestJSON(t, filepath.Join(dir, "input.json"), []string{"MixedCase"})
writeFile(t, filepath.Join(dir, "relspec.yml"), `version: 1
jobs:
lower:
command: convert
inputs:
- format: json
path: input.json
output:
format: json
path: output.json
options:
lowercase_identifiers: true
`)
set := mustLoadSet(t, filepath.Join(dir, "relspec.yml"))
if err := executeJobPlan(set, "lower", false, false, &bytes.Buffer{}); err != nil {
t.Fatal(err)
}
data, err := os.ReadFile(filepath.Join(dir, "output.json"))
if err != nil {
t.Fatal(err)
}
if !strings.Contains(string(data), "mixedcase") || strings.Contains(string(data), "MixedCase") {
t.Fatalf("%s", data)
}
}
+9
View File
@@ -24,6 +24,7 @@ import (
"git.warky.dev/wdevs/relspecgo/pkg/readers/sqlite" "git.warky.dev/wdevs/relspecgo/pkg/readers/sqlite"
"git.warky.dev/wdevs/relspecgo/pkg/readers/typeorm" "git.warky.dev/wdevs/relspecgo/pkg/readers/typeorm"
"git.warky.dev/wdevs/relspecgo/pkg/readers/yaml" "git.warky.dev/wdevs/relspecgo/pkg/readers/yaml"
"git.warky.dev/wdevs/relspecgo/pkg/transform"
"git.warky.dev/wdevs/relspecgo/pkg/writers" "git.warky.dev/wdevs/relspecgo/pkg/writers"
wbun "git.warky.dev/wdevs/relspecgo/pkg/writers/bun" wbun "git.warky.dev/wdevs/relspecgo/pkg/writers/bun"
wdbml "git.warky.dev/wdevs/relspecgo/pkg/writers/dbml" wdbml "git.warky.dev/wdevs/relspecgo/pkg/writers/dbml"
@@ -407,6 +408,14 @@ func readDatabaseForMerge(dbType, filePath, connString, label string) (*models.D
} }
func writeDatabaseForMerge(dbType, filePath, connString string, db *models.Database, label string, flattenSchema bool) error { func writeDatabaseForMerge(dbType, filePath, connString string, db *models.Database, label string, flattenSchema bool) error {
if lowercaseIdentifiers {
var err error
db, err = transform.LowercaseIdentifiers(db)
if err != nil {
return err
}
}
var writer writers.Writer var writer writers.Writer
switch strings.ToLower(dbType) { switch strings.ToLower(dbType) {
+2
View File
@@ -25,11 +25,13 @@ func newWriterOptions(outputPath, packageName string, flattenSchema bool, nullab
OutputPath: outputPath, OutputPath: outputPath,
PackageName: packageName, PackageName: packageName,
FlattenSchema: flattenSchema, FlattenSchema: flattenSchema,
LowercaseIdentifiers: lowercaseIdentifiers,
NullableTypes: nullableTypes, NullableTypes: nullableTypes,
NullableArrays: nullableArrays, NullableArrays: nullableArrays,
TypeMappings: typeMappings, TypeMappings: typeMappings,
Prisma7: prisma7, Prisma7: prisma7,
ContinueOnError: continueOnError, ContinueOnError: continueOnError,
StrictDirectives: strictDirectives, StrictDirectives: strictDirectives,
StrictConversion: strictConversion,
} }
} }
+7
View File
@@ -18,6 +18,8 @@ var (
noVersion bool noVersion bool
silent bool silent bool
strictDirectives bool strictDirectives bool
strictConversion bool
lowercaseIdentifiers bool
typeMapFlags []string typeMapFlags []string
typeMappings map[string]string typeMappings map[string]string
) )
@@ -39,12 +41,16 @@ JSON, YAML, SQL, etc.).`,
} }
func init() { func init() {
for _, cmd := range []*cobra.Command{convertCmd, batchCmd, mergeCmd, splitCmd, templCmd} {
cmd.Flags().BoolVar(&lowercaseIdentifiers, "lowercase", false, "Lowercase all model identifiers and structured references before output (SQL expressions and data keep their case)")
}
rootCmd.AddCommand(convertCmd) rootCmd.AddCommand(convertCmd)
rootCmd.AddCommand(batchCmd) rootCmd.AddCommand(batchCmd)
rootCmd.AddCommand(diffCmd) rootCmd.AddCommand(diffCmd)
rootCmd.AddCommand(inspectCmd) rootCmd.AddCommand(inspectCmd)
rootCmd.AddCommand(scriptsCmd) rootCmd.AddCommand(scriptsCmd)
rootCmd.AddCommand(jobCmd) rootCmd.AddCommand(jobCmd)
jobCmd.PersistentFlags().BoolVar(&lowercaseIdentifiers, "lowercase", false, "Lowercase all model identifiers and structured references before schema output")
rootCmd.AddCommand(assetsCmd) rootCmd.AddCommand(assetsCmd)
rootCmd.AddCommand(templCmd) rootCmd.AddCommand(templCmd)
rootCmd.AddCommand(editCmd) rootCmd.AddCommand(editCmd)
@@ -57,6 +63,7 @@ func init() {
rootCmd.PersistentFlags().StringArrayVar(&typeMapFlags, "type-map", nil, "Override a SQL-to-Go type mapping for bun/gorm output as sqltype=gotype (repeatable), e.g. --type-map uuid=uuid.UUID --type-map numeric=decimal.Decimal") rootCmd.PersistentFlags().StringArrayVar(&typeMapFlags, "type-map", nil, "Override a SQL-to-Go type mapping for bun/gorm output as sqltype=gotype (repeatable), e.g. --type-map uuid=uuid.UUID --type-map numeric=decimal.Decimal")
rootCmd.PersistentFlags().BoolVar(&noVersion, "no-version", false, "Suppress the RelSpec version header") rootCmd.PersistentFlags().BoolVar(&noVersion, "no-version", false, "Suppress the RelSpec version header")
rootCmd.PersistentFlags().BoolVar(&silent, "silent", false, "Suppress progress and status messages (errors are still shown)") rootCmd.PersistentFlags().BoolVar(&silent, "silent", false, "Suppress progress and status messages (errors are still shown)")
rootCmd.PersistentFlags().BoolVar(&strictConversion, "strict-conversion", false, "Reject known lossy MySQL schema conversions before writing or executing output")
rootCmd.PersistentFlags().BoolVar(&strictDirectives, "strict-directives", false, "Fail on unknown or untranslatable DBML dialect directives (@postgres:, @sqlite:, …)") rootCmd.PersistentFlags().BoolVar(&strictDirectives, "strict-directives", false, "Fail on unknown or untranslatable DBML dialect directives (@postgres:, @sqlite:, …)")
} }
+3 -1
View File
@@ -63,7 +63,7 @@ var scriptsExecuteCmd = &cobra.Command{
Long: `Execute SQL scripts from a directory against a PostgreSQL database. Long: `Execute SQL scripts from a directory against a PostgreSQL database.
Scripts are executed in order: Priority (ascending), Sequence (ascending), Name (alphabetical). Scripts are executed in order: Priority (ascending), Sequence (ascending), Name (alphabetical).
By default, execution stops immediately on the first error. Use --ignore-errors to continue execution. By default, execution stops immediately on the first error. Use --ignore-errors (or --continue-on-error / --coe) to attempt later scripts; any failure still exits nonzero.
The directory is scanned recursively for all subdirectories and files matching the patterns: The directory is scanned recursively for all subdirectories and files matching the patterns:
{priority}_{sequence}_{name}.sql or .pgsql (underscore format) {priority}_{sequence}_{name}.sql or .pgsql (underscore format)
@@ -112,6 +112,8 @@ func init() {
scriptsExecuteCmd.Flags().StringVar(&scriptsSchemaName, "schema", "public", "Schema name (optional, default: public)") scriptsExecuteCmd.Flags().StringVar(&scriptsSchemaName, "schema", "public", "Schema name (optional, default: public)")
scriptsExecuteCmd.Flags().StringVar(&scriptsDBName, "database", "database", "Database name (optional, default: database)") scriptsExecuteCmd.Flags().StringVar(&scriptsDBName, "database", "database", "Database name (optional, default: database)")
scriptsExecuteCmd.Flags().BoolVar(&scriptsIgnoreErrors, "ignore-errors", false, "Continue executing scripts even if errors occur") scriptsExecuteCmd.Flags().BoolVar(&scriptsIgnoreErrors, "ignore-errors", false, "Continue executing scripts even if errors occur")
scriptsExecuteCmd.Flags().BoolVar(&scriptsIgnoreErrors, "continue-on-error", false, "Attempt later scripts after failures; still exit nonzero")
scriptsExecuteCmd.Flags().BoolVar(&scriptsIgnoreErrors, "coe", false, "Alias for --continue-on-error")
err = scriptsExecuteCmd.MarkFlagRequired("dir") err = scriptsExecuteCmd.MarkFlagRequired("dir")
if err != nil { if err != nil {
+41
View File
@@ -0,0 +1,41 @@
package main
import (
"io"
"os"
"path/filepath"
"strings"
"testing"
"git.warky.dev/wdevs/relspecgo/pkg/jobs"
"git.warky.dev/wdevs/relspecgo/pkg/models"
)
func TestStrictConversionOptionAndDryRun(t *testing.T) {
prior := strictConversion
strictConversion = true
t.Cleanup(func() { strictConversion = prior })
if flag := rootCmd.PersistentFlags().Lookup("strict-conversion"); flag == nil {
t.Fatal("missing --strict-conversion")
}
db := models.InitDatabase("shop")
s := models.InitSchema("shop")
table := models.InitTable("users", "shop")
col := models.InitColumn("value", "users", "shop")
col.Type = "unknown_type"
table.Columns[col.Name] = col
s.Tables = append(s.Tables, table)
db.Schemas = append(db.Schemas, s)
if err := validateWriteTarget(db, "mysql", "", "", ""); err == nil || !strings.Contains(err.Error(), "lossy MySQL") {
t.Fatalf("dry run bypassed strict validation: %v", err)
}
strictConversion = false
out := filepath.Join(t.TempDir(), "out.sql")
rj := &resolvedJob{job: &jobs.Job{Output: &jobs.Output{Format: "mysql"}, Options: jobs.Options{StrictConversion: true}}, outputPath: out}
if err := writeJobOutput(rj, db, &jobLogger{file: io.Discard}); err == nil || !strings.Contains(err.Error(), "lossy MySQL") {
t.Fatalf("job option bypassed strict validation: %v", err)
}
if _, err := os.Stat(out); !os.IsNotExist(err) {
t.Fatalf("strict job created output: %v", err)
}
}
+8
View File
@@ -7,6 +7,7 @@ import (
"github.com/spf13/cobra" "github.com/spf13/cobra"
"git.warky.dev/wdevs/relspecgo/pkg/models" "git.warky.dev/wdevs/relspecgo/pkg/models"
"git.warky.dev/wdevs/relspecgo/pkg/transform"
"git.warky.dev/wdevs/relspecgo/pkg/writers" "git.warky.dev/wdevs/relspecgo/pkg/writers"
wtemplate "git.warky.dev/wdevs/relspecgo/pkg/writers/template" wtemplate "git.warky.dev/wdevs/relspecgo/pkg/writers/template"
) )
@@ -142,6 +143,13 @@ func runTempl(cmd *cobra.Command, args []string) error {
} }
} }
if lowercaseIdentifiers {
db, err = transform.LowercaseIdentifiers(db)
if err != nil {
return err
}
}
// Create template writer // Create template writer
fmt.Fprintf(os.Stderr, "Loading template: %s\n", templTemplatePath) fmt.Fprintf(os.Stderr, "Loading template: %s\n", templTemplatePath)
fmt.Fprintf(os.Stderr, "Execution mode: %s\n", templMode) fmt.Fprintf(os.Stderr, "Execution mode: %s\n", templMode)
+2
View File
@@ -191,6 +191,8 @@ jobs:
# (file output = full DDL); set options.full_ddl to force full DDL # (file output = full DDL); set options.full_ddl to force full DDL
overwrite: false # default false overwrite: false # default false
options: options:
strict_conversion: false # reject known lossy MySQL output before writing
lowercase_identifiers: false # lowercase names/references; SQL expressions and data keep their case
flatten_schema: false flatten_schema: false
schema: public schema: public
package: models # for gorm/bun output package: models # for gorm/bun output
+131
View File
@@ -0,0 +1,131 @@
# MySQL schema fidelity
The MySQL reader/writer preserves the table-schema features below. It generates
DDL for **new tables in existing databases**, not migrations or data copies.
MySQL 8.4 / InnoDB is exercised by the live suite. The reader uses MySQL 8.0.23+
metadata (including functional indexes, checks, spatial metadata, and invisible
columns); older MySQL versions and the `mariadb` alias are not certified by this
suite and may return explicit metadata-query errors.
## Conversion diagnostics
MySQL output warns on stderr when it encounters known unsupported features.
Add `--strict-conversion` to reject these conversions before opening an output
file or connecting to an output database:
```bash
relspec convert --from mysql --from-conn "$MYSQL_DSN" \
--to mysql --to-path schema.sql --strict-conversion
relspec convert --from json --from-path schema.json \
--to mysql --to-path schema.sql --strict-conversion --dry-run
```
The flag also applies to MySQL output from `batch`, `split`, and schema jobs.
Jobs can set `options.strict_conversion: true`. Library callers use
`writers.WriterOptions.StrictConversion`; `Warning func(string)` collects
non-strict diagnostics instead of printing them. `mysql.Writer.Statements`
performs the same preflight without opening a destination and returns structured
SQL statements suitable for execution without splitting on semicolons.
Strict conversion currently covers **MySQL output**. It is not a general
losslessness check for other writers or a validator of arbitrary SQL fragments.
Warnings identify the object and omitted or changed feature. Inspect them before
using non-strict output; unsupported index types are omitted, while a partial
index's predicate and INCLUDE columns cannot be represented in MySQL.
## MySQL capability matrix
| Feature | Reader → model | Model → MySQL DDL |
| --- | --- | --- |
| Ordered columns, nullability, primary/composite keys | Preserved | Preserved |
| Unique/composite keys | Preserved, including backing indexes | Preserved once; no duplicate unique index |
| Foreign/composite keys and DELETE/UPDATE actions | Preserved | Added after all tables, allowing forward references/cycles |
| CHECK and NOT ENFORCED CHECK | Preserved | Preserved |
| BTREE, FULLTEXT, SPATIAL indexes | Preserved | Emitted; engine restrictions still apply |
| Expression/prefix/descending/invisible index parts | Preserved in `Index.KeyParts` and metadata | Preserved |
| HASH indexes | Preserved | Engine-dependent; incompatible engine produces a diagnostic |
| Integer unsignedness; mediumint; enum/set; binary/blob/text variants | Canonical type plus native declaration | Native declaration preserved |
| Length, numeric precision/scale, fractional seconds | Typed dimensions plus native declaration | Preserved; canonical dimensions also supported |
| Defaults, including SQL expressions, bit/binary literals and multiline text | Values plus native SQL spelling from SHOW CREATE | Preserved |
| Auto-increment and current next value | Preserved | Preserved |
| Stored/virtual generated columns, ON UPDATE, invisible columns | Preserved | Preserved |
| Engine, table/column collation, charset and comments | Preserved | Preserved |
| Views, triggers, routines, events, partitioned tables | Names recorded in `mysql_unsupported` | Diagnostic; bodies/partition definitions are not emitted |
| Nonempty table CREATE_OPTIONS; native parser/storage/engine attributes; spatial SRID | Recorded as unsupported attributes | Diagnostic; not emitted |
| Partial/INCLUDE indexes, unsupported index methods | Retained when supplied in the model | Diagnostic |
| Deferrable constraints, identity semantics, timestamptz, jsonb storage | Retained when supplied in the model | Diagnostic for unsupported/different semantics |
| Standalone sequences/enums/domains, ownership/permissions, scripts and dialect directives | Retained when supplied in the model | Diagnostic; not emitted |
| Data, grants, users, server/plugin configuration | Outside the reader's schema contract | Outside the writer's contract |
Diagnostics depend on the objects visible to the connection account. They cover
known modeled/discovered features, not every engine/plugin extension. The server
still validates SQL expressions, engine rules, and references at execution time.
For example, source and target engines must support the constraints requested.
## Preservation across intermediate formats
| Path | Scope |
| --- | --- |
| MySQL → model → MySQL | Live-tested for the features above on MySQL 8.4 / InnoDB |
| MySQL → JSON → model → MySQL | Live-tested; preserves typed index parts and MySQL metadata |
| MySQL → YAML → model → MySQL | YAML serializes the same fields/metadata; covered by the live round-trip test |
| Other source/target formats | Their own mappings apply; MySQL-specific metadata and semantics are not certified to survive |
Native declarations and defaults are stored in column metadata as `mysql_type`
and `mysql_default_sql`. Reader-generated signatures (`mysql_type_signature`,
`mysql_default_value`) prevent these spellings from silently overriding edited
types/dimensions/defaults. Editing canonical dimensions invalidates the native
type signature, produces a diagnostic, and uses the canonical mapping instead.
For a binary default, the logical default is its SQL hex/literal spelling so it
survives serialization without invalid UTF-8.
Other metadata includes `mysql_charset`, `mysql_generation_storage`,
`mysql_default_expression`, `mysql_on_update`, `mysql_invisible`, table
`mysql_engine` / `mysql_collation` / `mysql_auto_increment`, index
`mysql_visible`, and check `mysql_enforced`. Unknown `mysql_*` metadata produces
a diagnostic. For a manually authored expression default, set
`mysql_default_expression: true`; otherwise strings are literal values.
`Index.KeyParts` is authoritative when present; `Index.Columns` remains the
compatibility representation for simple indexes.
Raw native type declarations, default SQL, generation/check/index expressions,
and ON UPDATE expressions are executable SQL input. Lowercasing changes names
and structured index parts but leaves expressions untouched; update references
inside raw SQL yourself when using `--lowercase`.
## Applying the output
Create the destination database/schema first. Qualified names are preserved,
including foreign-key target schemas. Adjust them explicitly when restoring to
a different database. Output assumes standard MySQL backslash escaping;
`NO_BACKSLASH_ESCAPES` or `ANSI_QUOTES` on the source produces a conversion
diagnostic, and live execution refuses these target modes before DDL. File consumers must use the same
supported mode. Other SQL-mode incompatibilities surface as server errors.
`CREATE TABLE IF NOT EXISTS` does not reconcile existing definitions. Foreign
keys use `ALTER TABLE ADD CONSTRAINT` and therefore repeated application can
fail on existing constraints. This is creation DDL, not a rerunnable migration.
Live execution stops on the first SQL error by default. Explicit
`ContinueOnError` attempts later statements and still returns an error if any
fail. MySQL DDL implicitly commits; `AtomicExecution` is rejected. PostgreSQL's
shared execution-report support is not yet implemented for this writer.
## Verification
```bash
./tests/dbtest/dbtest.sh test mysql
# Or use your own disposable database:
RELSPEC_TEST_MYSQL_CONN="$MYSQL_DSN" go test -mod=vendor ./pkg/readers/mysql ./pkg/writers/mysql
```
The live tests compare MySQL's `SHOW CREATE TABLE` definitions, normalizing only
temporary target-name suffixes and nonsemantic ordinary-index display order.
They re-read the target, exercise constraint enforcement/default values, and
verify unsupported objects produce warnings or strict failures without touching
an existing output file. Test fixtures are removed after each run.
Metadata references: [COLUMNS](https://dev.mysql.com/doc/refman/8.4/en/information-schema-columns-table.html),
[STATISTICS](https://dev.mysql.com/doc/refman/8.4/en/information-schema-statistics-table.html),
[TABLE_CONSTRAINTS](https://dev.mysql.com/doc/refman/8.4/en/information-schema-table-constraints-table.html),
and [REFERENTIAL_CONSTRAINTS](https://dev.mysql.com/doc/refman/8.4/en/information-schema-referential-constraints-table.html).
+30 -2
View File
@@ -96,11 +96,39 @@ INSERT INTO assets (message, payload)
VALUES (:message, decode(:payload, 'base64')::bytea); VALUES (:message, decode(:payload, 'base64')::bytea);
``` ```
- `path`: File path resolved relative to the SQL file containing the directive - `path`: Relative file path beneath the SQL file’s directory; traversal, absolute paths, and symlinks are rejected
- `var`: Named placeholder to replace, such as `:message` - `var`: Named placeholder to replace, such as `:message`
- `mode`: `text` embeds an escaped SQL string literal; `base64` embeds a base64 string literal - `mode`: `text` embeds an escaped SQL string literal; `base64` embeds a base64 string literal
The directive comment is removed from the SQL, and every matching placeholder is replaced before the script is listed or executed. A directive must be a standalone SQL line comment. Directive-like text inside
quoted strings, quoted identifiers, block comments, or dollar-quoted bodies is
ordinary SQL content. The directive is removed, and every matching placeholder in
executable SQL is replaced once before listing or execution. Replacements are
literal, preserving `$1`, `${2}`, `$HOME`, and other dollar text; inserted content
is never scanned for another placeholder. Duplicate embed variables or variables
used only inside strings/comments are errors. PostgreSQL casts (`::type`) remain
unchanged. Asset-call named parameters follow these same lexical boundaries;
repeated parameters share one positional argument.
Embed and manifest asset reads share root-scoped containment and a 64 MiB per-file
limit. Only regular files are accepted. The SQL directory reader uses its selected
script tree as an explicit asset root; embed paths remain relative to each script.
Symlinks in asset paths are rejected, and containment is enforced again at open time.
Manifest entries are checked at discovery and execution, so replacing a file with an
escaping symlink after scanning cannot expose an external file. No per-directive
external-file override is supported; copy required assets into the allowed tree.
Library callers can choose a trusted root and size limit with
`assetloader.ProcessEmbedDirectivesWithOptions(..., assetloader.EmbedOptions{
AssetRoot: root, MaxFileBytes: limit})`; zero means the 64 MiB default, and negative
or unbounded limits are rejected. Manifest items retain `AssetRoot` and
`MaxFileBytes` for execution-time validation.
For scripts loaded from a model rather than a SQL directory, the executor requires
trusted writer configuration `Metadata["asset_root"]` before expanding directives.
Model `source_path` metadata supplies resolution context only; it does not grant
local-file access. Missing or out-of-root source paths fail before reading assets.
## Commands ## Commands
+10
View File
@@ -570,3 +570,13 @@ COMMIT;
- [RelSpec Documentation](../README.md) - [RelSpec Documentation](../README.md)
- [Model Structure Reference](../pkg/models/) - [Model Structure Reference](../pkg/models/)
- [Example Templates](../examples/templates/) - [Example Templates](../examples/templates/)
## Output path safety
In schema, domain, script, and table modes, `--output` always names a directory,
including a nonexistent directory without a trailing slash. An empty value uses
the current directory. Database mode uses a single output file as before.
Generated paths must stay relative to that directory. Traversal, absolute or
Windows-specific unsafe paths, symlinks, filename collisions (including case
variants), and file/directory collisions are rejected before publishing artifacts.
Nested patterns such as `{{.ParentSchema.Name}}/{{.Name}}.txt` remain supported.
+359
View File
@@ -0,0 +1,359 @@
# Project audit 1
Date: 2026-10-12
Revision reviewed: `bfe98528265b45be7bfdcdf025710981bae06a7b`
Perspectives: software engineering, security, maintenance, and databases.
## Repair progress
Implementation batches (2026-10-12):
- [x] **A01, PostgreSQL failure propagation:** stop on the first SQL failure by default; explicit continuation still returns a structured `ExecutionFailure`. Reports include skipped entries, and requested report write/close failures are errors. Live tests cover a middle-statement failure in both modes and verify dependent jobs do not start.
- [x] **A02, implicit replacement removed:** both full-DDL and diff type changes propagate conversion errors without renaming/replacing the column. Live populated-data tests verify original values, column count/type, unique-index identity, and constraint enforcement survive incompatible conversions.
- [x] **Related defect exposed by A01:** full-DDL column additions now use native `ADD COLUMN IF NOT EXISTS`, fixing duplicate-column errors for flattened output that previously appeared successful.
- [x] **A01, shared execution results:** PostgreSQL, SQLite, SQL Server, and raw script execution now use `writers.ExecutionReport` / `ExecutionFailure`. Stop/continue modes always fail on SQL errors; counters/results survive early failure and aggregate across script schemas. Reports distinguish applied, failed, skipped, rolled-back, and uncertain operations. Generated DDL supports optional atomic execution; raw scripts report script-managed outcomes without promising statement-level commit/rollback. Requested report failures are errors in every executor. Live PostgreSQL, SQL Server, SQLite, and dependent-job regressions pass.
- [x] **A02, conversion expressions and preflight:** column metadata `pgsql_using` configures an explicit SQL conversion expression; `pgsql_preflight` adds a row predicate. Full-DDL and diff plans evaluate every existing value before schema mutations, and live preflight failure stops even with continuation enabled. Tests cover later invalid rows, preservation of a constraint scheduled for removal, custom conversions, predicate rejection, invalid configuration, and block-delimiter collisions. Standalone schema generation also includes preflights.
- [x] **A03, contained template output:** multi-file output has an explicit directory root, portable relative filename validation, and preflight collision/symlink checks. Rendering/validation complete before writing; exclusive per-file staging and root-scoped rename prevent generated filenames from escaping the root. Multi-file atomic publication remains A16 work.
- [x] **A04, contained asset reads:** embed directives and manifest execution use shared root-scoped reads, symlink checks, regular-file validation, and a bounded 64 MiB default. Manifest paths are rechecked at execution. Imported source-path metadata cannot authorize file reads without a trusted configured asset root. Library callers can explicitly configure a root/size limit; directives cannot opt out of containment.
- [x] **A05, PostgreSQL quoting:** schema/full-DDL/diff paths and embedded templates always quote model identifiers, quote literal/catalog/regclass values, sanitize names in generated line comments, and choose collision-safe dollar delimiters for procedural and audit function bodies. NUL is rejected before output; migration comparisons preserve identifier case. Raw SQL expressions remain an explicit executable-input boundary.
- [x] **Requested lowercase option:** `--lowercase` on convert/batch/merge/split/templ/job run, job `options.lowercase_identifiers`, and PostgreSQL `WriterOptions.LowercaseIdentifiers` normalize names and structured references on a copy, rejecting collisions. SQL expressions, defaults, comments, enum values, and data keep their case; callers must adjust references inside raw SQL themselves.
- [x] **A08, SQL-aware substitution/execution:** embeds use one-pass literal substitution, preserving dollar characters and preventing replacement rescanning. Named parameters and embed directives respect SQL strings, quoted identifiers, nested comments, casts, and dollar bodies; duplicate/unused embed variables fail. MySQL executes structured generated statements without splitting multiline literals or stripping their contents.
- [x] **A06, nullable MySQL catalogs:** constraint references, functional-index columns/expressions, index prefix/direction fields, column dimensions/collations/generation metadata, and optional table auto-increment counters use nullable destinations. Queries qualify joined columns. Live primary/unique/composite/functional-key reads pass, including unsigned auto-increment values beyond signed 64-bit range.
- [x] **A07, scoped MySQL fidelity and diagnostics:** native types/default spellings and typed dimensions/index parts survive MySQL → model → JSON/YAML → DDL. Output emits foreign keys/actions after all tables, checks/enforcement, unique/ordinary/functional/prefix/descending/invisible/fulltext indexes, generated columns, comments/collations/charsets, ON UPDATE, invisible columns, and auto-increment counters. Known unsupported objects/attributes produce warnings or `--strict-conversion` / job `options.strict_conversion` / library `StrictConversion` failures before output or connection. See [the scoped capability matrix](MYSQL_FIDELITY.md); this covers new-table DDL on MySQL 8.4 / InnoDB, not all engines, MariaDB versions, other format mappings, or migrations.
A06–A07 verification (2026-10-12): live MySQL 8.4.11 tests compare source/target `SHOW CREATE TABLE` definitions (normalizing temporary target suffixes and ordinary-index display order), re-read the target, and verify FK/check enforcement and binary/bit/multiline/backslash/default values. The suite covers composite/functional unique keys, reverse table dependency order, generated/check/index expressions containing SQL literals, precision, unsigned ranges, and multiple FKs to the same target. It exposed and fixed truncated binary defaults in `information_schema` by extracting native defaults from SQL-aware `SHOW CREATE TABLE` definitions. Native SHOW CREATE also avoids stale auto-increment statistics, and escaped catalog expressions are decoded before rendering. Unsupported views/triggers/partitioned tables produce warnings and strict failures; strict failure preserves existing files and happens before opening an output connection. CLI dry-run and per-job strict options are tested. The MySQL container test runner now selects reader/writer packages by default. `go test -mod=vendor -p 1 ./...` passed with `RELSPEC_TEST_MYSQL_CONN` configured; `go vet -mod=vendor ./...`, CLI strict/dry-run checks, shell syntax, and `git diff --check` passed. The MySQL container runner also passed using its schema-scoped account; its disposable server enables trigger-fixture creation after final-server readiness. PostgreSQL and SQL Server live variables were unset for this batch. Checks used `go1.27.1-X:nodwarf5 linux/amd64`; declared-minimum-toolchain coverage remains A13 work. Disposable MySQL instances were removed after verification. A08 now also has live MySQL coverage for embedded statement delimiters/default contents.
A05–A08 verification: unit regressions and live PostgreSQL 18 checks cover unusual names, comments/defaults, rerunnable file/statement output, diff migrations, sequences, primary-key replacement, audit functions/triggers, and both standard-string modes. MySQL statement preservation is unit-tested; a live MySQL instance was not used for A08. `go test -mod=vendor -p 1 ./...` passed with `RELSPEC_TEST_PG_CONN` set to the disposable instance; `go vet -mod=vendor ./...`, executable CLI help checks, and `git diff --check` passed. SQL Server tests were not configured for this batch. The disposable instance was removed after verification.
Additional observation: a parallel live full-suite run hit a PostgreSQL reader catalog race (`could not open relation with OID`) while another package's test dropped schema objects. The writer safely fell back to full DDL and its preflight still stopped all mutations, but the existing test assumed no successful read-only preflights before failure. Running live packages serially passed. Reader resilience to concurrent catalog changes remains outside A05/A08; A10 specifically concerns cancellation/timeouts.
A03–A04 verification: `go test -mod=vendor ./...`, targeted regression tests, `go vet -mod=vendor ./...`, and `git diff --check` passed. Database-dependent tests were not configured for this filesystem-only batch. Windows test binaries were cross-compiled for the template and asset packages; Windows runtime filesystem tests remain unperformed.
Verification: `go test -mod=vendor ./...` passed with `RELSPEC_TEST_PG_CONN` and `RELSPEC_TEST_MSSQL_CONN` pointing to disposable PostgreSQL 18 and SQL Server 2022 Developer instances; SQLite regressions use temporary database files. A minimal public table was seeded for the existing PostgreSQL `ReadTable` test, which requires a nonempty default schema. `go vet -mod=vendor ./...` and `git diff --check` passed. Executable CLI checks verified `scripts execute --ignore-errors`, `--continue-on-error`, and `--coe` all attempt a later script after failure and exit 1. Raw script tests intentionally make no statement-level rollback claim. A09 remains open for concurrency, history, and migration recovery; sequence/external side effects are outside transaction rollback guarantees. Original audit evidence is retained for historical context.
## Assessment
RelSpec has a useful architecture, broad format support, and substantial tests. Its most serious weaknesses are in the boundary between generating a schema and safely applying that schema to a real database. A successful process exit does not reliably mean a successful migration. Some conversions silently discard database semantics, and several file/SQL generation paths do not preserve the safety guarantees implemented elsewhere.
**Recommendation: fix the high-priority findings before relying on unattended production database writes or processing externally supplied specifications.** Continue using generated output with review and disposable database validation while these issues are addressed. This is a repository audit, not a certification of production readiness.
### Existing strengths
- Readers, writers, the intermediate model, diffing, inspection, and transformations are separated into packages.
- There are many unit and regression tests, including live PostgreSQL writer tests and SQLite tests.
- Job files have semantic validation, dependency-cycle detection, environment-based connection references, path containment checks, and symlink checks.
- Single-file job conversions already use temporary output followed by rename.
- The asset manifest executor binds file bytes as database parameters.
- The repository includes a private vulnerability reporting policy, vendored dependencies, and release-time vet/static analysis/vulnerability-check steps.
These controls should be extended and made consistent, rather than replaced wholesale.
## Scope and verification
Reviewed the CLI, job handling, database readers/writers, PostgreSQL templates and migration generation, asset embedding, generic templates, connection builder, updater, build/release workflows, documentation, and the VS Code extension. Review depth was greatest for database mutation and input/output trust boundaries; every parser and format combination was not exhaustively audited.
Checks performed:
| Check | Result |
| --- | --- |
| `GOCACHE=/tmp/relspec-audit-gocache go test -mod=vendor ./...` | Initial sandbox run failed because HTTP tests could not open loopback sockets. Approved rerun outside the sandbox passed. |
| `GOCACHE=/tmp/relspec-audit-gocache go vet -mod=vendor ./...` | Passed. |
| `make fmt-check GOLANGCI_LINT=false` | Incorrectly passed, confirming that a failed formatter can be hidden. |
| Temporary Go reproduction using exported package APIs | Confirmed template path escape, embed path escape, embed dollar corruption, invalid PostgreSQL identifier/literal rendering, omitted MySQL FK/index output, and placeholder matching inside SQL strings. |
The local compiler reported `go1.27.1-X:nodwarf5 linux/amd64`; `go.mod` requires `1.26.9`. This was not a verification under the declared minimum toolchain. No live PostgreSQL, MySQL, or SQL Server connection was configured for the audit. Tests requiring those connections skipped; a passing package does not demonstrate live database correctness. Race testing, fuzz campaigns, benchmarks, `staticcheck`, and a fresh `govulncheck` scan were not performed. No claims about specific dependency CVEs are made.
Reproduction files and logs were placed in `/tmp`; the only intended repository change is this document. No real credentials or user data were used in reproductions. Database injection and migration effects were assessed from code/generated SQL, not executed against a database.
### Priority and evidence notation
- **P1 / High:** address before unattended database use or handling less-trusted inputs; credible data-integrity, confidentiality, or integrity impact.
- **P2 / Medium:** schedule in the next maintenance cycle; operational reliability, reproducibility, or defense-in-depth issue.
- **P3 / Low:** documentation or developer-experience issue.
- **Reproduced:** a local executable check demonstrated the stated behavior.
- **Code-confirmed:** the implementation establishes the behavior, but the complete external-system scenario was not executed.
## Findings
### A01 — PostgreSQL writes report success after statement failures
**Repair status: addressed.** The shared executor now enforces stop/continue behavior and nonzero failure across PostgreSQL, SQLite, SQL Server, and raw scripts, including requested report errors. Reports include individual outcomes, skipped operations, confirmed rollbacks, and uncertain finalization. Atomic generated-DDL runs are optional; raw scripts explicitly report script-managed execution. A09 remains a separate migration-management/concurrency contract.
**P1 / High · Engineering, database · Code-confirmed**
Evidence: `pkg/writers/pgsql/writer.go:2200–2322`, especially `executeStatements` and `finishReport`; `pkg/writers/pgsql/live_execute_test.go:144` explicitly tests continuing after failure.
Every failed statement is recorded and execution continues. `finishReport` prints an error count but returns `nil`. Failure to write the requested report is also only a warning. This behavior is unconditional in this execution path. A deployment or dependent job can proceed after an incomplete schema change because the writer returned success.
**Fix:** return a structured error when any statement fails; stop on the first error by default. A continue-on-error option should control continuation, separately from the final exit status. Aggregate results consistently across PostgreSQL, SQLite, SQL Server, and script execution. Treat failure to produce an explicitly requested execution report as an error.
**Acceptance:** force a middle statement to fail; the CLI exits nonzero, dependent jobs do not run, and the report distinguishes applied, failed, skipped, and rolled-back operations. Test both stop and continue modes.
### A02 — Failed type conversion silently substitutes an empty column
**Repair status: addressed.** The shared type-change template propagates conversion errors. Configurable `pgsql_using` expressions and `pgsql_preflight` predicates are supported, with automatic whole-column cast evaluation before schema mutations. Full-DDL and diff live tests verify preservation of populated columns and unique dependencies, custom conversions, and rejected predicates. Preflight is advisory under concurrent writes; atomic generated-DDL execution is available for transactionally supported changes, and separately reviewed rename/replacement remains required.
**P1 / High · Database · Code-confirmed**
Evidence: `pkg/writers/pgsql/templates/alter_column_type_with_check.tmpl:18–27`; `pkg/writers/pgsql/migration_writer.go:491`; `pkg/writers/pgsql/writer.go:514`.
The type-change block catches `WHEN OTHERS`, renames the existing column, and adds a fresh column with the original name. For example, converting populated text values containing non-numeric data to an integer can fail the cast and activate this fallback. The application then sees an empty replacement column. Existing indexes, constraints, and dependencies may remain attached to the renamed column. Original values are not necessarily deleted, but the application's expected column no longer contains them.
The broad exception handler also treats unrelated failures as permission to attempt this schema rewrite.
**Fix:** fail on conversion errors. Require a separately reviewed migration for any rename-and-replace operation, including backfill, validation, dependency handling, and recovery. Support an explicit conversion expression and preflight checks where appropriate.
**Acceptance:** migrating populated incompatible data leaves the original column, values, and dependencies intact and returns an actionable failure. No rename/replacement occurs implicitly.
### A03 — Generated template filenames escape the output directory
**Repair status: addressed.** Multi-file roots, generated paths, collisions, symlinks, and root-scoped writes are covered by regression tests. A nonexistent output path now consistently denotes the directory itself rather than its parent.
**P1 / High · Security · Reproduced**
Evidence: `pkg/writers/template/writer.go:249–308`.
The default filename pattern uses `{{.Name}}.txt`. `generateFilename` joins the rendered name with the destination without checking containment; `writeOutput` creates directories and writes the resulting path. A schema named `../escaped` wrote `escaped.txt` beside the selected output directory in the reproduction.
An imported model can therefore cause writes outside the intended destination, including overwriting files writable by the current user. Job-level checks validate the declared output path, not every subsequently generated filename. Exploitation requires the user to process the supplied model in a multi-file template mode.
**Fix:** validate every rendered filename against a clearly defined output root; reject traversal, unsafe absolute paths, and symlink escapes. Detect filename collisions before writing. Prefer root-scoped filesystem operations so validation and opening cannot be separated by a symlink replacement.
**Acceptance:** malicious names, filename patterns, symlink parents, duplicate names, and platform-specific paths cannot write outside the root or silently overwrite another generated artifact.
### A04 — SQL embed directives can read outside the script directory
**Repair status: addressed.** Shared bounded root-scoped reads protect directives and manifest assets, including revalidation after discovery. Model source-path metadata alone no longer grants local file access.
**P1 / High · Security · Reproduced**
Evidence: `pkg/assetloader/embed.go:73–74`; compare the containment checks in `pkg/assetloader/loader.go` and `pkg/jobs/jobs.go`.
Embed paths are cleaned and joined to the script directory, then passed directly to `os.ReadFile`. A directive using `path=../synthetic-secret.txt` successfully embedded a sibling file. There is no equivalent root or symlink policy for this path. Script metadata also supplies the source path used for resolution.
A supplied script can cause local readable files to be copied into generated SQL or sent to the destination database. Permission to execute SQL against a database should not implicitly grant unrestricted local-file access. This is a local input trust-boundary issue, not an unauthenticated network endpoint.
**Fix:** define an explicit asset root, contain all embed reads within it, check symlink resolution, limit file size, and document any opt-in allowance for external files. Reuse the policy across manifest assets and embed directives.
**Acceptance:** traversal and symlink escape fail before reading; valid in-root text and binary assets still work; diagnostics do not print file contents.
### A05 — PostgreSQL SQL quoting is incomplete across syntactic contexts
**P1 / High · Security, database · Reproduced rendering defects**
Evidence: `pkg/writers/pgsql/template_functions.go:134`; `pkg/writers/pgsql/templates/add_column_with_check.tmpl:5–7`; `create_foreign_key_with_check.tmpl`; `alter_column_type_with_check.tmpl`.
Two separate defects were confirmed:
1. `quoteIdent` does not actually check reserved keywords, despite its comment. The template executor generated `CREATE TABLE IF NOT EXISTS public.order (select text);` for legal names requiring quoting.
2. Catalog lookups insert names directly into single-quoted SQL strings. A table named `O'Reilly` generated `table_name = 'O'Reilly'`. Quoting a name for an identifier elsewhere does not escape it as a string literal here.
Fixed `DO $$ ... $$` delimiters create another context that must be protected from embedded delimiter text. Besides breaking valid database names, insufficient context-aware quoting is an injection risk when less-trusted metadata is rendered and executed using privileged database credentials. A complete injection exploit was not executed during this audit. Raw SQL expressions are a separate, intentional executable-input boundary and should be documented as such.
**Fix:** centralize identifier, literal, qualified-name, and block-delimiter handling. Always quote identifiers where practical; parameterize values where supported. Generate collision-safe dollar delimiters or avoid interpolating arbitrary content into fixed blocks. Apply this across all templates, not only the examples above.
**Acceptance:** generated SQL executes correctly for reserved words, apostrophes, double quotes, Unicode, newlines, and dollar delimiters, and cannot introduce additional SQL statements through a name.
PostgreSQL documents the distinct identifier, string, and dollar-quote rules in [Lexical Structure](https://www.postgresql.org/docs/current/sql-syntax-lexical.html).
### A06 — MySQL constraint reading scans nullable metadata into strings
**Repair status: addressed.** Nullable metadata scans and qualified queries are covered by live MySQL 8.4 round trips with primary, unique, composite FK, and functional-index definitions.
**P1 / High · Database · Code-confirmed; live MySQL not exercised**
Evidence: `pkg/readers/mysql/reader.go:181–220`, specifically the `rs`, `rt`, and `rc` string variables passed to `rows.Scan`.
The constraint query reads `REFERENCED_TABLE_SCHEMA`, `REFERENCED_TABLE_NAME`, and `REFERENCED_COLUMN_NAME` for primary and unique constraints as well as foreign keys. Non-foreign-key rows have no referenced object and can contain SQL NULL. Scanning these into Go strings fails, aborting inspection of ordinary tables with a primary key.
**Fix:** use nullable destinations such as `sql.NullString`, or explicit `COALESCE` where an empty string has the intended meaning. Review other nullable metadata, including expression-index column names, in the same reader.
**Acceptance:** live MySQL tests read tables with primary, unique, composite foreign-key, and expression-index definitions without scan failures and preserve their metadata. The current reader tests only cover type mapping and a missing connection string.
Reference: [MySQL KEY_COLUMN_USAGE documentation](https://dev.mysql.com/doc/refman/8.0/en/information-schema-key-column-usage-table.html) and [MySQL's documented non-FK NULL behavior](https://bugs.mysql.com/bug.php?id=9587).
### A07 — MySQL conversion silently loses constraints, indexes, and type detail
**Repair status: addressed within the [documented MySQL table-schema contract](MYSQL_FIDELITY.md).** Supported features survive live round trips through JSON/YAML; known unsupported modeled/discovered features produce diagnostics or strict conversion failures. MySQL output remains creation DDL for existing databases, with engine/version/SQL-mode limits stated in the capability matrix.
**P1 / High · Database · Reproduced omissions; additional losses code-confirmed**
Evidence: `pkg/writers/mysql/writer.go:96–163`; `pkg/readers/mysql/reader.go:153–178`; `pkg/mariadb/datatypes.go`.
The writer emits columns, primary keys, and unique constraints, but does not emit foreign keys, checks, or ordinary indexes from the model. A reproduction containing both a foreign key and an index returned success while emitting neither. The reader maps `COLUMN_TYPE` to a canonical base type without populating the separate length/precision/scale fields; its tests explicitly map `bigint unsigned` to `int64`.
Consequences include lost referential integrity, missing performance-critical indexes, changed ranges, and changed string/decimal definitions. Advertising a format as supported does not communicate these losses.
**Fix:** preserve and emit supported semantics; return explicit diagnostics for every unsupported object or attribute. Add a strict conversion mode that refuses lossy output and a per-format capability matrix. Preserve unsignedness, dimensions, generated expressions, and FK actions through the intermediate representation.
**Acceptance:** inspect → model → DDL → disposable database → inspect tests compare semantic equivalence for representative MySQL schemas. Unsupported features produce visible, testable failures or warnings, never silent success.
### A08 — Embed substitution corrupts dollar characters; placeholder parsing ignores SQL syntax
**P2 / Medium · Engineering, database · Reproduced**
Evidence: `pkg/assetloader/embed.go:159–160`; `pkg/assetloader/executor.go:32–87`.
`replaceEmbedPlaceholder` passes file content into the replacement argument of `regexp.ReplaceAllString`. Dollar characters are interpreted as capture references. Embedding `price $1 ${2} $HOME $$` produced `price ; $` in the local reproduction, rather than preserving the input.
Separately, `BuildQuery` matches placeholders inside quoted strings and comments. `SELECT ':literal', :bytes` fails with an unknown-placeholder error even though `:literal` is SQL string content. Similar text-based processing makes the MySQL executor's `strings.Split(..., ";\n")` fragile for multiline defaults containing that sequence.
**Fix:** use literal replacement callbacks or explicit span assembly for embed content. Parse SQL lexical contexts for named placeholders. Execute a structured list of generated statements instead of splitting SQL text on delimiters.
**Acceptance:** dollar text, repeated placeholders, quoted strings, comments, casts, dollar-quoted bodies, and embedded statement delimiters preserve exact intended semantics.
### A09 — Migration execution lacks a coherent atomicity and concurrency contract
**P1 / High · Database, engineering · Code-confirmed**
Evidence: `pkg/writers/pgsql/writer.go:2200–2300`; `pkg/writers/sqlexec/writer.go:107–222`; `pkg/writers/sqlite/writer.go:118–164`; `pkg/writers/mysql/writer.go:165–188`.
Executors issue independent statements/scripts without a transaction spanning the operation, migration-wide lock, or applied-script ledger in these paths. A failure can leave earlier work committed. Re-running script execution repeats supplied scripts; concurrent invocations can interleave. A script may contain its own transaction, but that does not establish a consistent executor-level guarantee.
The necessary behavior differs by database and statement type: wrapping all MySQL DDL in a transaction would not provide the same rollback guarantee as transactional PostgreSQL DDL.
**Fix:** define explicit execution modes with transaction boundaries, concurrency control, retry/recovery semantics, and accurate partial-application reports. If `scripts execute` remains a raw runner, document that clearly. If it is intended as a migration manager, add migration identity, checksums, applied history, and duplicate/concurrent-run protection.
**Acceptance:** failure injection and simultaneous-run tests establish predictable behavior; supported transactional runs roll back; nontransactional runs report exactly what committed and how to resume safely.
### A10 — Database operations cannot consistently inherit cancellation or deadlines
**P2 / Medium · Engineering, operations · Code-confirmed**
Evidence: `pkg/readers/reader.go`; `pkg/writers/writer.go`; `pkg/writers/pgsql/writer.go:2205`; `pkg/writers/sqlexec/writer.go`; `pkg/readers/mysql/reader.go:25`; `pkg/writers/mysql/writer.go:184`.
Reader/writer interfaces do not accept a caller context, and several implementations start with `context.Background()`. This disconnects CLI/job cancellation from connection establishment, catalog reads, and SQL execution. A blocked statement or lock wait has no consistent application-level deadline.
**Fix:** carry caller contexts through reader/writer operations; configure connection, statement, and lock-wait limits separately. Use a signal-aware CLI root context and preserve cancellation errors in job results.
**Acceptance:** a blocked database query terminates promptly on cancellation or timeout, cleanup is bounded, and the job exits nonzero without starting dependents.
### A11 — Windows updater executes downloads without artifact authentication
**P2 / Medium · Security, supply chain · Code-confirmed**
Evidence: `cmd/relspec/update.go:115–152`; `.gitea/workflows/release.yml`.
After confirmation or `--yes`, the updater downloads the asset URL from release metadata and starts the resulting executable. There is no signature verification, trusted-publisher check, content-size limit, or asset-origin/redirect restriction in this path. The default release API uses HTTPS, which is a useful transport control, but it is not independent artifact authentication. Compromised release metadata or publication infrastructure could substitute an installer.
**Fix:** sign installers or release manifests and verify against an independently trusted identity before launch. Require HTTPS throughout redirects, apply a deliberate origin policy, cap downloads, and clean up failed downloads. A checksum fetched from the same compromised endpoint alone is insufficient authentication.
**Acceptance:** tampered, unsigned, wrong-publisher, oversized, and disallowed-origin assets are rejected before execution.
### A12 — Connection and development-service defaults permit avoidable exposure
**P2 / Medium · Security · Code-confirmed**
Evidence: `pkg/ui/connstring.go:42–49`; `docker-compose.yml`; `Makefile`'s `docker-up` target.
The UI starts PostgreSQL and SQL Server connection fields with encryption disabled; changing the host does not change that setting. Compose publishes database ports without a loopback host restriction and supplies fixed development passwords. These are development credentials, not evidence of leaked production secrets, but the services can become reachable on host network interfaces.
**Fix:** default remote connections to authenticated encrypted transport and make local exceptions explicit. Bind development database ports to `127.0.0.1` by default; document how to deliberately expose a test service. Prefer a limited inspection role over administrative defaults.
**Acceptance:** changing a connection to a remote host does not silently retain an insecure default; standard local test startup only exposes loopback ports.
### A13 — CI does not consistently test the declared toolchain or live writer paths
**P1 / High · Maintenance, database assurance · Code-confirmed configuration defects**
Evidence: `.github/workflows/ci.yml:15,58,79`; `.github/workflows/integration-tests.yml:59–76`; `.gitea/workflows/release.yml:48–49`; `Makefile:92–108`; `tests/dbtest/dbs/mysql.sh`.
- GitHub CI specifies Go 1.24/1.25 while `go.mod` requires 1.26.9. Automatic toolchain switching can make that matrix misleading; with switching disabled those jobs cannot build the module as declared.
- The GitHub integration job runs directly on the host but connects to `postgres`, the service-container name, rather than the published loopback port. GitHub documents `localhost` for this job layout. This networking conclusion applies to GitHub-hosted execution; the Gitea runner configuration was not inspected.
- `make test-integration` runs the PostgreSQL reader and `tests/integration`, but omits live tests under `pkg/writers/pgsql` and `pkg/writers/sqlexec`. Release `go test ./...` has no configured database service/connection, so those tests can skip there too.
- MySQL has container scaffolding but no Go tests consuming `RELSPEC_TEST_MYSQL_CONN`; this leaves A06 undetected.
- `make test-unit` excludes the entire PostgreSQL reader package, including `queries_pure_test.go`.
**Fix:** use `go-version-file: go.mod`, explicitly test supported toolchains, correct GitHub service networking, and run all live readers and writers against disposable databases in required CI jobs. Make unexpected integration-test skips fail the integration job. Verify which workflows the actual Gitea deployment discovers; CI badges alone do not establish execution.
**Acceptance:** CI artifacts show real reader/writer execution, not just green packages with skipped live tests, and demonstrate MySQL constraint inspection and failing-migration behavior.
Reference: [GitHub PostgreSQL service-container networking](https://docs.github.com/en/actions/tutorials/use-containerized-services/create-postgresql-service-containers).
### A14 — Formatting gate ignores formatter failure
**P2 / Medium · Maintenance · Reproduced**
Evidence: `Makefile:62–66`.
The recipe captures formatter stdout and checks whether it is empty, without preserving the command's failure status. `make fmt-check GOLANGCI_LINT=false` prints `Checking formatting...` and exits successfully. A missing/broken formatter or dependency/toolchain failure can therefore pass this gate without checking formatting.
**Fix:** explicitly fail when the formatter command fails before inspecting its diff output. Pin the formatter version so the gate is reproducible.
**Acceptance:** a formatter that exits nonzero with empty stdout fails the target; clean formatting passes; a real diff fails.
### A15 — Release artifacts can disagree with the requested release tag
**P2 / Medium · Maintenance, supply chain · Code-confirmed**
Evidence: `.gitea/workflows/release.yml:55–94,241–269,297–314`; upload steps around lines 113–130, 278–289, and 360–371; `Makefile:84,143,256,273`.
For manual dispatch with a tag input, checkout does not explicitly select that tag. Binaries and packages are built from the dispatched checkout, while the source archives use the supplied tag and version strings claim that version. These can represent different commits. Uploads use `curl -s` without failing on HTTP error status, so publication failures can also look successful.
Additional reproducibility gaps: ordinary `make build` runs `go mod tidy` through `deps`; analysis tools use `@latest`; `release-version` lacks test/build prerequisites; `rerelease` force-moves an existing release tag.
**Fix:** resolve and validate the release tag once, check out its commit in every artifact job, and record that commit in a manifest. Fail on HTTP errors and verify the uploaded inventory. Keep normal builds read-only with respect to dependency manifests, pin build tools, and publish immutable versions with consistent quality gates.
**Acceptance:** dispatching a historical tag from a newer branch produces binaries and source from the same historical commit; failed uploads fail the workflow; normal builds leave the working tree unchanged.
### A16 — Output replacement guarantees differ across commands and writers
**P2 / Medium · Engineering, maintenance · Code-confirmed**
Evidence: `cmd/relspec/job.go:859–886`; `pkg/writers/template/writer.go:288–308`; `pkg/writers/mysql/writer.go:54–68`; `pkg/writers/json/writer.go:56–58`.
Job single-file conversion has an atomic-write wrapper, but direct writers and template output write or truncate final paths in place. Some writer cleanup functions ignore close errors. A disk error, failed generation, or interrupted multi-file run can replace a previously valid artifact with incomplete output or leave a mixed generation.
**Fix:** centralize output staging and finalization for direct CLI use and jobs. Handle close/flush errors, use exclusive unpredictable temporary files, preflight filename collisions, and define a staging/manifest strategy for multi-file generations. Preserve existing files when generation fails.
**Acceptance:** injected generation/write/finalization failures leave the previous valid output intact and return an error; no command claims success with a partial replacement.
## Perspective-specific maintenance needs
These are follow-up investments and scope gaps, not additional proven vulnerabilities.
### Software engineering
- **A common capability registry:** format construction is repeated across `convert.go`, `diff.go`, `merge.go`, `inspect.go`, and `edit.go`. Job format validation is another list. Centralize reader/writer creation and capabilities so adding a format does not require synchronized switches throughout the CLI.
- **Typed execution options and results:** connection settings, execution counters, and flags are carried through string-keyed metadata maps. Introduce typed configuration/result structures for live execution while retaining metadata for extensions.
- **Model validation before generation:** JSON/YAML readers primarily unmarshal and return models. Establish a shared normalization/validation boundary for nil entries, duplicate objects, unresolved references, incompatible attributes, and absent required fields. The inspector is useful but should not be the only optional path to structural safety.
- **Smaller database writer components:** `pkg/writers/pgsql/writer.go` is 2,643 lines and combines generation, execution, reporting, type/index policy, and statement parsing. Separate these responsibilities to make security and migration changes easier to review.
- **Adversarial parser tests:** no `Fuzz...` tests were found in `pkg` or `cmd`. Add fuzz targets for DBML/directives, ORM readers, embed handling, SQL lexical processing, and invalid intermediate models. Add large-schema benchmarks before introducing concurrency or caching.
### Security
- Document the trust model: SQL scripts, expressions/defaults, asset calls, schema metadata, templates, and release assets have different execution and local-file capabilities.
- Test redaction across every CLI/library logging path, not just the job logger and masked UI preview. Cover URL-encoded passwords, DSN parse errors, and database error messages without printing real credentials in tests.
- Run pinned vulnerability checks in required change-validation workflows and record the tool/database versions. Produce release checksums and an SBOM; signing and verification are covered by A11.
- Use scoped database roles: inspection should not require DDL rights, and schema application should not normally use a server administrator account.
### Maintenance
- Update contradictory documentation. README advertises Go 1.24.0; CONTRIBUTING says 1.25+; `go.mod` requires 1.26.9. TODO says MySQL, watch mode, type mappings, Windows installer, and installer CI are missing even though implementations exist. `tests/dbtest/README.md` still says there is no MySQL driver/reader.
- Document actual feature completeness, not just format names. Distinguish implemented, partial, experimental, and unsupported behavior with links to tests.
- The VS Code extension's `renderTemplate` is a placeholder and syntax validation mainly counts delimiters per line (`vscode-extension/relspec-template-editor/src/extension.ts`). Label it experimental or connect it to the actual Go parser/renderer. Reuse and dispose a single diagnostics collection rather than creating one on each validation.
- Align GitHub and Gitea workflow ownership, tool versions, required checks, release policies, and developer setup. Add Windows execution tests for filesystem behavior and installer/updater code, rather than relying solely on cross-compilation.
### Database specialization
- Publish an object/attribute fidelity matrix covering every supported source/destination, including views, sequences, enums, generated/identity columns, FK actions/deferral, expression/partial indexes, precision/scale, unsigned types, and permissions.
- PostgreSQL generation needs explicit fidelity review beyond tables: the model includes views, enums, permissions, and standalone sequences, but the main PostgreSQL writer does not provide corresponding complete emission paths. Its sequence generation focuses on table primary keys; scanning views to infer extensions does not emit the views. Unsupported objects need warnings or rejection.
- Establish migration safety policy: destructive-change approval in the product, reviewed plans, populated-data preflight, lock budgets, backfills, recovery, and post-apply verification. A dry run that only validates options is not proof that a migration can execute safely.
- Test semantic convergence: after applying a model, re-inspect and diff; a second application should generate no unintended change. Include populated tables, cross-schema dependencies, generated columns, reserved names, and failure recovery.
- Keep database-specific transaction and DDL behavior explicit. Do not promise uniform rollback or schema support across PostgreSQL, SQLite, SQL Server, and MySQL.
## Prioritized repair plan
| Order | Work | Completion evidence |
| --- | --- | --- |
| 1 | Fix misleading success and unsafe type fallback: A01–A02 | Failure-injection tests prove nonzero exit and preservation of existing data/schema. |
| 2 | Close file and SQL rendering boundaries: A03–A05, A08 | Adversarial filename, embed, identifier, literal, and delimiter cases pass. |
| 3 | Repair MySQL reading and fidelity: A06–A07 | Live round trips preserve constraints, indexes, and type dimensions; unsupported features fail explicitly. |
| 4 | Restore trustworthy CI coverage: A13–A14 | All relevant live writers run, toolchain matches policy, and tool failures fail gates. |
| 5 | Define execution safety: A09–A10 | Documented transaction/concurrency modes, cancellation, recovery, and precise results. |
| 6 | Harden distribution and output: A11–A12, A15–A16 | Verified installer artifacts, consistent release commit, safer defaults, and staged file replacement. |
| 7 | Reduce maintenance debt | Capability registry, typed options, accurate docs, supported-feature matrix, fuzzing, and measured performance baseline. |
Avoid expanding the format list or adding a web/API layer until the existing execution and fidelity contracts are dependable. The current code already offers enough functionality that correctness, safety, and maintainability will yield more value than additional breadth.
+24
View File
@@ -0,0 +1,24 @@
# Maintainer: Hein (Warky Devs) <hein@warky.dev>
pkgname=relspec-bin
pkgver=1.0.91
pkgrel=1
pkgdesc="RelSpec is a comprehensive database relations management tool that reads, transforms, and writes database table specifications across multiple formats and ORMs."
arch=('x86_64' 'aarch64')
url="https://git.warky.dev/wdevs/relspecgo"
license=('MIT')
provides=('relspec')
conflicts=('relspec')
options=('!debug')
source_x86_64=("$pkgname-$pkgver-x86_64.tar.gz::$url/releases/download/v$pkgver/relspec_${pkgver}_linux_amd64.tar.gz")
source_aarch64=("$pkgname-$pkgver-aarch64.tar.gz::$url/releases/download/v$pkgver/relspec_${pkgver}_linux_arm64.tar.gz")
sha256sums_x86_64=('SKIP')
sha256sums_aarch64=('SKIP')
package() {
# Binary
install -Dm755 "relspec" "$pkgdir/usr/bin/relspec"
install -Dm644 "LICENSE" "$pkgdir/usr/share/licenses/$pkgname/LICENSE"
# Default config dir
install -dm755 "$pkgdir/etc/relspec"
}
@@ -1,6 +1,6 @@
# Maintainer: Hein (Warky Devs) <hein@warky.dev> # Maintainer: Hein (Warky Devs) <hein@warky.dev>
pkgname=relspec pkgname=relspec
pkgver=1.0.89 pkgver=1.0.91
pkgrel=1 pkgrel=1
pkgdesc="RelSpec is a comprehensive database relations management tool that reads, transforms, and writes database table specifications across multiple formats and ORMs." pkgdesc="RelSpec is a comprehensive database relations management tool that reads, transforms, and writes database table specifications across multiple formats and ORMs."
arch=('x86_64' 'aarch64') arch=('x86_64' 'aarch64')
+1 -1
View File
@@ -1,5 +1,5 @@
Name: relspec Name: relspec
Version: 1.0.89 Version: 1.0.91
Release: 1%{?dist} Release: 1%{?dist}
Summary: RelSpec is a comprehensive database relations management tool that reads, transforms, and writes database table specifications across multiple formats and ORMs. Summary: RelSpec is a comprehensive database relations management tool that reads, transforms, and writes database table specifications across multiple formats and ORMs.
+94 -19
View File
@@ -9,10 +9,22 @@ import (
"strconv" "strconv"
"strings" "strings"
"unicode/utf8" "unicode/utf8"
"git.warky.dev/wdevs/relspecgo/pkg/safefile"
"git.warky.dev/wdevs/relspecgo/pkg/sqllex"
) )
const ScriptSourcePathMetadataKey = "source_path" const ScriptSourcePathMetadataKey = "source_path"
const DefaultMaxAssetBytes int64 = 64 << 20 // 64 MiB per manifest/embedded asset
// EmbedOptions are trusted caller configuration, not script/model metadata.
// An empty AssetRoot defaults to the SQL file's directory; external reads are rejected.
type EmbedOptions struct {
AssetRoot string
MaxFileBytes int64
}
var ( var (
embedDirectivePattern = regexp.MustCompile(`(?m)^\s*--\s*@embed:\s*(.+?)\s*$`) embedDirectivePattern = regexp.MustCompile(`(?m)^\s*--\s*@embed:\s*(.+?)\s*$`)
embedAttrPattern = regexp.MustCompile(`([a-zA-Z_][a-zA-Z0-9_]*)=("[^"]*"|'[^']*'|\S+)`) embedAttrPattern = regexp.MustCompile(`([a-zA-Z_][a-zA-Z0-9_]*)=("[^"]*"|'[^']*'|\S+)`)
@@ -27,7 +39,28 @@ var (
// string literal. Base64 mode embeds a quoted base64 literal suitable for // string literal. Base64 mode embeds a quoted base64 literal suitable for
// decode(:var, 'base64'). // decode(:var, 'base64').
func ProcessEmbedDirectives(sqlPath, sql string) (string, error) { func ProcessEmbedDirectives(sqlPath, sql string) (string, error) {
directives := embedDirectivePattern.FindAllStringSubmatch(sql, -1) return ProcessEmbedDirectivesWithOptions(sqlPath, sql, EmbedOptions{})
}
func embedDirectives(sql string) (result string, directives []string) {
// Only standalone line comments are directives, and only in SQL code.
result = sqllex.Walk(sql, nil, func(comment string) string {
if match := embedDirectivePattern.FindStringSubmatch(comment); match != nil {
directives = append(directives, match[1])
return ""
}
return comment
})
return result, directives
}
func HasEmbedDirectives(sql string) bool {
_, directives := embedDirectives(sql)
return len(directives) > 0
}
func ProcessEmbedDirectivesWithOptions(sqlPath, sql string, options EmbedOptions) (string, error) {
result, directives := embedDirectives(sql)
if len(directives) == 0 { if len(directives) == 0 {
return sql, nil return sql, nil
} }
@@ -36,22 +69,63 @@ func ProcessEmbedDirectives(sqlPath, sql string) (string, error) {
return "", fmt.Errorf("sql path is required for embed directives") return "", fmt.Errorf("sql path is required for embed directives")
} }
result := embedDirectivePattern.ReplaceAllString(sql, "") assetRoot := options.AssetRoot
for i, directive := range directives { if assetRoot == "" {
literal, placeholder, err := embedDirectiveLiteral(sqlPath, directive[1], i+1) assetRoot = filepath.Dir(sqlPath)
}
absRoot, err := filepath.Abs(assetRoot)
if err != nil { if err != nil {
return "", err return "", err
} }
if !embedPlaceholderPattern(placeholder).MatchString(result) { absSQL, err := filepath.Abs(sqlPath)
return "", fmt.Errorf("%s embed directive %d: placeholder %s not found", sqlPath, i+1, placeholder) if err != nil {
return "", err
}
scriptRel, err := filepath.Rel(absRoot, absSQL)
if err != nil {
return "", err
}
if _, err := safefile.Relative(filepath.ToSlash(scriptRel)); err != nil {
return "", fmt.Errorf("SQL source path is outside asset root: %w", err)
}
root, err := openAssetRoot(absRoot)
if err != nil {
return "", err
}
defer root.Close()
limit := options.MaxFileBytes
if limit == 0 {
limit = DefaultMaxAssetBytes
}
literals := map[string]string{}
used := map[string]bool{}
for i, directive := range directives {
literal, placeholder, err := embedDirectiveLiteral(root, filepath.Dir(scriptRel), limit, sqlPath, directive, i+1)
if err != nil {
return "", err
}
if _, exists := literals[placeholder]; exists {
return "", fmt.Errorf("duplicate embed placeholder %s", placeholder)
}
literals[placeholder] = literal
}
result = sqllex.Walk(result, func(name string) string {
if literal, ok := literals[name]; ok {
used[name] = true
return literal
}
return name
}, nil)
for name := range literals {
if !used[name] {
return "", fmt.Errorf("%s: embed placeholder %s not found in executable SQL", sqlPath, name)
} }
result = replaceEmbedPlaceholder(result, placeholder, literal)
} }
return result, nil return result, nil
} }
func embedDirectiveLiteral(sqlPath, raw string, directiveNumber int) (literal, placeholder string, err error) { func embedDirectiveLiteral(root *os.Root, scriptDir string, maxBytes int64, sqlPath, raw string, directiveNumber int) (literal, placeholder string, err error) {
attrs, err := parseEmbedAttrs(raw) attrs, err := parseEmbedAttrs(raw)
if err != nil { if err != nil {
return "", "", fmt.Errorf("%s embed directive %d: %w", sqlPath, directiveNumber, err) return "", "", fmt.Errorf("%s embed directive %d: %w", sqlPath, directiveNumber, err)
@@ -70,8 +144,12 @@ func embedDirectiveLiteral(sqlPath, raw string, directiveNumber int) (literal, p
return "", "", fmt.Errorf("%s embed directive %d: mode must be text or base64", sqlPath, directiveNumber) return "", "", fmt.Errorf("%s embed directive %d: mode must be text or base64", sqlPath, directiveNumber)
} }
resolved := filepath.Join(filepath.Dir(sqlPath), filepath.Clean(pathValue)) relative, err := safefile.Relative(pathValue)
data, err := os.ReadFile(resolved) if err != nil {
return "", "", fmt.Errorf("%s embed directive %d: %w", sqlPath, directiveNumber, err)
}
resolved := filepath.Join(scriptDir, relative)
data, err := safefile.Read(root, resolved, maxBytes)
if err != nil { if err != nil {
return "", "", fmt.Errorf("%s embed directive %d: reading %s: %w", sqlPath, directiveNumber, resolved, err) return "", "", fmt.Errorf("%s embed directive %d: reading %s: %w", sqlPath, directiveNumber, resolved, err)
} }
@@ -153,13 +231,10 @@ func sqlStringLiteral(value string) (string, error) {
if strings.ContainsRune(value, '\x00') { if strings.ContainsRune(value, '\x00') {
return "", fmt.Errorf("embedded text contains NUL byte") return "", fmt.Errorf("embedded text contains NUL byte")
} }
return "'" + strings.ReplaceAll(value, "'", "''") + "'", nil prefix := ""
} if strings.Contains(value, "\\") {
prefix = "E"
func replaceEmbedPlaceholder(sql, placeholder, literal string) string { value = strings.ReplaceAll(value, "\\", "\\\\")
return embedPlaceholderPattern(placeholder).ReplaceAllString(sql, "${1}"+literal+"${2}") }
} return prefix + "'" + strings.ReplaceAll(value, "'", "''") + "'", nil
func embedPlaceholderPattern(placeholder string) *regexp.Regexp {
return regexp.MustCompile(`(^|[^a-zA-Z0-9_:])` + regexp.QuoteMeta(placeholder) + `([^a-zA-Z0-9_]|$)`)
} }
+111
View File
@@ -141,3 +141,114 @@ SELECT :body, :body_extra;
t.Fatalf("placeholder boundary was not respected:\n%s", got) t.Fatalf("placeholder boundary was not respected:\n%s", got)
} }
} }
func TestEmbedRejectsTraversalAbsolutePathsAndSymlinks(t *testing.T) {
root := t.TempDir()
scriptDir := filepath.Join(root, "scripts")
if err := os.Mkdir(scriptDir, 0o755); err != nil {
t.Fatal(err)
}
secret := filepath.Join(root, "secret.txt")
if err := os.WriteFile(secret, []byte("never print this secret"), 0o600); err != nil {
t.Fatal(err)
}
for _, path := range []string{"../secret.txt", secret, `..\secret.txt`, `C:\secret.txt`, `\\server\secret.txt`} {
sql := "-- @embed: path='" + path + "' var=:body mode=text\nSELECT :body;"
got, err := ProcessEmbedDirectives(filepath.Join(scriptDir, "seed.sql"), sql)
if err == nil || got != "" {
t.Fatalf("unsafe embed path accepted %q: %q %v", path, got, err)
}
if strings.Contains(err.Error(), "never print this secret") {
t.Fatal("diagnostic leaked asset bytes")
}
}
if err := os.Symlink(root, filepath.Join(scriptDir, "link")); err != nil {
t.Skip(err)
}
if _, err := ProcessEmbedDirectives(filepath.Join(scriptDir, "seed.sql"), "-- @embed: path=link/secret.txt var=:body mode=text\nSELECT :body;"); err == nil {
t.Fatal("symlink parent must be rejected")
}
if err := os.Symlink(secret, filepath.Join(scriptDir, "leaf.txt")); err != nil {
t.Fatal(err)
}
if _, err := ProcessEmbedDirectives(filepath.Join(scriptDir, "seed.sql"), "-- @embed: path=leaf.txt var=:body mode=text\nSELECT :body;"); err == nil {
t.Fatal("symlink asset must be rejected")
}
}
func TestEmbedExplicitRootAndSizeLimit(t *testing.T) {
root := t.TempDir()
dir := filepath.Join(root, "scripts")
if err := os.Mkdir(dir, 0o755); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filepath.Join(dir, "body.txt"), []byte("12345"), 0o600); err != nil {
t.Fatal(err)
}
sql := "-- @embed: path=body.txt var=:body mode=text\nSELECT :body;"
path := filepath.Join(dir, "seed.sql")
if _, err := ProcessEmbedDirectivesWithOptions(path, sql, EmbedOptions{AssetRoot: root, MaxFileBytes: 4}); err == nil || !strings.Contains(err.Error(), "byte limit") {
t.Fatalf("expected size limit error: %v", err)
}
if got, err := ProcessEmbedDirectivesWithOptions(path, sql, EmbedOptions{AssetRoot: root, MaxFileBytes: 5}); err != nil || !strings.Contains(got, "SELECT '12345';") {
t.Fatalf("valid bounded embed: %q %v", got, err)
}
if _, err := ProcessEmbedDirectivesWithOptions(path, sql, EmbedOptions{AssetRoot: t.TempDir()}); err == nil {
t.Fatal("source path outside explicit root must fail")
}
if _, err := ProcessEmbedDirectivesWithOptions(path, sql, EmbedOptions{MaxFileBytes: -1}); err == nil {
t.Fatal("negative limits must fail")
}
}
func TestManifestReadRechecksContainmentAndSize(t *testing.T) {
root, external := t.TempDir(), t.TempDir()
file := filepath.Join(root, "body.txt")
if err := os.WriteFile(file, []byte("valid"), 0o600); err != nil {
t.Fatal(err)
}
item := Item{AssetRoot: root, MaxFileBytes: 4, Entry: ManifestEntry{File: file}}
if _, err := readItemAsset(item); err == nil {
t.Fatal("manifest size bound must apply at execution")
}
item.MaxFileBytes = 5
if got, err := readItemAsset(item); err != nil || string(got) != "valid" {
t.Fatalf("valid manifest asset: %q %v", got, err)
}
secret := filepath.Join(external, "secret.txt")
if err := os.WriteFile(secret, []byte("secret"), 0o600); err != nil {
t.Fatal(err)
}
if err := os.Remove(file); err != nil {
t.Fatal(err)
}
if err := os.Symlink(secret, file); err != nil {
t.Skip(err)
}
if _, err := readItemAsset(item); err == nil {
t.Fatal("replacement symlink after scanning must fail")
}
item.Entry.File = secret
if _, err := readItemAsset(item); err == nil {
t.Fatal("forged asset path must fail")
}
}
func TestEmbedDefaultLimitRejectsOversizedFile(t *testing.T) {
dir := t.TempDir()
file, err := os.Create(filepath.Join(dir, "large.bin"))
if err != nil {
t.Fatal(err)
}
if err := file.Truncate(DefaultMaxAssetBytes + 1); err != nil {
file.Close()
t.Fatal(err)
}
if err := file.Close(); err != nil {
t.Fatal(err)
}
_, err = ProcessEmbedDirectives(filepath.Join(dir, "seed.sql"), "-- @embed: path=large.bin var=:payload mode=base64\nSELECT :payload;")
if err == nil || !strings.Contains(err.Error(), "byte limit") {
t.Fatalf("expected default size bound: %v", err)
}
}
+5 -18
View File
@@ -3,20 +3,13 @@ package assetloader
import ( import (
"context" "context"
"fmt" "fmt"
"os"
"path/filepath" "path/filepath"
"regexp"
"strings" "git.warky.dev/wdevs/relspecgo/pkg/sqllex"
"github.com/jackc/pgx/v5" "github.com/jackc/pgx/v5"
) )
// namedPlaceholder matches :identifier patterns (but not ::cast syntax).
var namedPlaceholder = regexp.MustCompile(`:([a-zA-Z_][a-zA-Z0-9_]*)`)
// pgCastMarker temporarily replaces :: to protect PostgreSQL cast syntax.
const pgCastMarker = "\x00PGCAST\x00"
// BuildQuery converts a SQL call that uses :name named placeholders into a // BuildQuery converts a SQL call that uses :name named placeholders into a
// pgx-compatible positional-parameter query ($1, $2, …) and returns the // pgx-compatible positional-parameter query ($1, $2, …) and returns the
// corresponding argument slice. // corresponding argument slice.
@@ -30,13 +23,10 @@ const pgCastMarker = "\x00PGCAST\x00"
// placeholder (not built-in and not in staticParams) returns an error. // placeholder (not built-in and not in staticParams) returns an error.
// PostgreSQL cast syntax (::type) is left untouched. // PostgreSQL cast syntax (::type) is left untouched.
func BuildQuery(call string, fileBytes []byte, filename string, staticParams map[string]string) (query string, args []any, err error) { func BuildQuery(call string, fileBytes []byte, filename string, staticParams map[string]string) (query string, args []any, err error) {
// Protect :: casts before running the placeholder regex.
protected := strings.ReplaceAll(call, "::", pgCastMarker)
paramIndex := map[string]int{} // name → 1-based position paramIndex := map[string]int{} // name → 1-based position
var firstErr error var firstErr error
query = namedPlaceholder.ReplaceAllStringFunc(protected, func(match string) string { query = sqllex.Walk(call, func(match string) string {
if firstErr != nil { if firstErr != nil {
return match return match
} }
@@ -70,15 +60,12 @@ func BuildQuery(call string, fileBytes []byte, filename string, staticParams map
paramIndex[name] = idx paramIndex[name] = idx
args = append(args, val) args = append(args, val)
return fmt.Sprintf("$%d", idx) return fmt.Sprintf("$%d", idx)
}) }, nil)
if firstErr != nil { if firstErr != nil {
return "", nil, firstErr return "", nil, firstErr
} }
// Restore :: casts.
query = strings.ReplaceAll(query, pgCastMarker, "::")
return query, args, nil return query, args, nil
} }
@@ -86,7 +73,7 @@ func BuildQuery(call string, fileBytes []byte, filename string, staticParams map
// absolute path set by ScanDir) and executes the configured SQL call via conn. // absolute path set by ScanDir) and executes the configured SQL call via conn.
// The file's raw bytes are bound as a []byte parameter — no encoding or escaping. // The file's raw bytes are bound as a []byte parameter — no encoding or escaping.
func ExecuteItem(ctx context.Context, conn *pgx.Conn, item Item) error { func ExecuteItem(ctx context.Context, conn *pgx.Conn, item Item) error {
data, err := os.ReadFile(item.Entry.File) data, err := readItemAsset(item)
if err != nil { if err != nil {
return fmt.Errorf("reading asset file %s: %w", item.Entry.File, err) return fmt.Errorf("reading asset file %s: %w", item.Entry.File, err)
} }
+36
View File
@@ -0,0 +1,36 @@
package assetloader
import (
"fmt"
"os"
"path/filepath"
"git.warky.dev/wdevs/relspecgo/pkg/safefile"
)
func openAssetRoot(path string) (*os.Root, error) {
if err := safefile.CheckRoot(path); err != nil {
return nil, err
}
return os.OpenRoot(path)
}
func readItemAsset(item Item) ([]byte, error) {
if item.AssetRoot == "" {
return nil, fmt.Errorf("asset root is required for manifest execution")
}
root, err := openAssetRoot(item.AssetRoot)
if err != nil {
return nil, err
}
defer root.Close()
relative, err := filepath.Rel(item.AssetRoot, item.Entry.File)
if err != nil {
return nil, err
}
limit := item.MaxFileBytes
if limit == 0 {
limit = DefaultMaxAssetBytes
}
return safefile.Read(root, relative, limit)
}
+29 -21
View File
@@ -18,9 +18,10 @@ import (
"regexp" "regexp"
"sort" "sort"
"strconv" "strconv"
"strings"
"gopkg.in/yaml.v3" "gopkg.in/yaml.v3"
"git.warky.dev/wdevs/relspecgo/pkg/safefile"
) )
// ManifestEntry describes a single file to load from an assets.yaml manifest. // ManifestEntry describes a single file to load from an assets.yaml manifest.
@@ -37,6 +38,9 @@ type ManifestEntry struct {
// Item combines a manifest entry with its ordering metadata and the resolved // Item combines a manifest entry with its ordering metadata and the resolved
// directory where the manifest and asset file reside. // directory where the manifest and asset file reside.
type Item struct { type Item struct {
// AssetRoot is the trusted scan root, retained for validation at execution time.
AssetRoot string
MaxFileBytes int64
// Priority and Sequence come from the parent directory's naming pattern. // Priority and Sequence come from the parent directory's naming pattern.
Priority int Priority int
Sequence uint Sequence uint
@@ -57,7 +61,12 @@ var dirPattern = regexp.MustCompile(`^(\d+)[_-](\d+)[_-](.+)$`)
// absent or contains invalid YAML. // absent or contains invalid YAML.
func LoadManifest(dir string) ([]ManifestEntry, error) { func LoadManifest(dir string) ([]ManifestEntry, error) {
manifestPath := filepath.Join(dir, "assets.yaml") manifestPath := filepath.Join(dir, "assets.yaml")
data, err := os.ReadFile(manifestPath) root, err := openAssetRoot(dir)
if err != nil {
return nil, err
}
defer root.Close()
data, err := safefile.Read(root, "assets.yaml", DefaultMaxAssetBytes)
if err != nil { if err != nil {
return nil, fmt.Errorf("reading %s: %w", manifestPath, err) return nil, fmt.Errorf("reading %s: %w", manifestPath, err)
} }
@@ -92,7 +101,7 @@ func ScanDir(baseDir string) ([]Item, error) {
if d.IsDir() { if d.IsDir() {
return nil return nil
} }
if d.Name() != "assets.yaml" { if d.Type()&os.ModeSymlink != 0 || d.Name() != "assets.yaml" {
return nil return nil
} }
@@ -119,6 +128,8 @@ func ScanDir(baseDir string) ([]Item, error) {
} }
items = append(items, Item{ items = append(items, Item{
AssetRoot: absBase,
MaxFileBytes: DefaultMaxAssetBytes,
Priority: priority, Priority: priority,
Sequence: sequence, Sequence: sequence,
DirName: dirName, DirName: dirName,
@@ -170,31 +181,28 @@ func parseDirName(dir string) (priority int, sequence uint, name string) {
// Returns the absolute path, a skip flag (true when the entry should be silently // Returns the absolute path, a skip flag (true when the entry should be silently
// dropped), and any hard error. // dropped), and any hard error.
func resolveAssetPath(absBase, manifestDir, file string) (absPath string, skip bool, err error) { func resolveAssetPath(absBase, manifestDir, file string) (absPath string, skip bool, err error) {
// Clean and join before any symlink resolution so we can detect traversal. if _, err := safefile.Relative(file); err != nil {
joined := filepath.Join(manifestDir, filepath.Clean(file)) return "", true, nil
}
// Ensure the cleaned path is still inside absBase. joined := filepath.Join(manifestDir, file)
rel, err := filepath.Rel(absBase, joined) rel, err := filepath.Rel(absBase, joined)
if err != nil || strings.HasPrefix(rel, "..") { if err != nil {
// Path escapes the base directory; skip silently.
return "", true, nil return "", true, nil
} }
root, err := openAssetRoot(absBase)
// Walk each component to detect symlinks. if err != nil {
parts := strings.Split(rel, string(filepath.Separator)) return "", false, err
current := absBase }
for _, part := range parts { defer root.Close()
current = filepath.Join(current, part) if err := safefile.Check(root, rel); err != nil {
info, statErr := os.Lstat(current)
if statErr != nil {
// File doesn't exist; skip.
return "", true, nil return "", true, nil
} }
if info.Mode()&os.ModeSymlink != 0 { info, err := root.Stat(rel)
// Symlink in path; skip silently. if err != nil || !info.Mode().IsRegular() {
return "", true, nil return "", true, nil
} }
if info.Size() > DefaultMaxAssetBytes {
return "", false, fmt.Errorf("asset %q exceeds %d byte limit", rel, DefaultMaxAssetBytes)
} }
return joined, false, nil return joined, false, nil
} }
+60
View File
@@ -0,0 +1,60 @@
package assetloader
import (
"os"
"path/filepath"
"strings"
"testing"
)
func TestBuildQuerySQLContexts(t *testing.T) {
call := `SELECT ':unknown', ":unknown", $$:unknown$$, E'it\':unknown', :bytes::bytea, :filename, :bytes /* :unknown /* :unknown */ */ -- :unknown
`
got, args, err := BuildQuery(call, []byte("$1"), "File.txt", nil)
if err != nil {
t.Fatal(err)
}
if len(args) != 2 || !strings.Contains(got, "$1::bytea, $2, $1") || !strings.Contains(got, "$$:unknown$$") {
t.Fatalf("%s %#v", got, args)
}
}
func TestEmbedLiteralOnePassAndSQLContexts(t *testing.T) {
dir := t.TempDir()
data := "price $1 ${2} $HOME $$ :b 'quote' \\path"
for file, text := range map[string]string{"a.txt": data, "b.txt": "Second"} {
if err := os.WriteFile(filepath.Join(dir, file), []byte(text), 0o600); err != nil {
t.Fatal(err)
}
}
sql := `-- @embed: path=a.txt var=:a mode=text
-- @embed: path=b.txt var=:b mode=text
SELECT :a,:a,:b, ':a', ":a", $$:a$$, 1::integer; -- :a
/* :a */`
got, err := ProcessEmbedDirectives(filepath.Join(dir, "script.sql"), sql)
if err != nil {
t.Fatal(err)
}
literal, err := sqlStringLiteral(data)
if err != nil {
t.Fatal(err)
}
if strings.Count(got, literal) != 2 || !strings.Contains(got, "'Second'") || !strings.Contains(got, "':a'") || !strings.Contains(got, "$$:a$$") {
t.Fatalf("%s", got)
}
for _, sql := range []string{"SELECT $body$\n-- @embed: path=missing var=:x mode=text\n$body$", "SELECT '\n-- @embed: path=missing var=:x mode=text\n'", "/*\n-- @embed: path=missing var=:x mode=text\n*/", "SELECT 1; -- @embed: path=missing var=:x mode=text"} {
if HasEmbedDirectives(sql) {
t.Fatalf("false directive: %s", sql)
}
got, err := ProcessEmbedDirectives("", sql)
if err != nil || got != sql {
t.Fatalf("%q: %v", got, err)
}
}
for _, sql := range []string{"-- @embed: path=a.txt var=:a mode=text\nSELECT ':a'", "-- @embed: path=a.txt var=:a mode=text\n-- @embed: path=b.txt var=:a mode=text\nSELECT :a"} {
if _, err := ProcessEmbedDirectives(filepath.Join(dir, "script.sql"), sql); err == nil {
t.Fatal("expected invalid directive use")
}
}
}
+11
View File
@@ -277,6 +277,8 @@ func parseHumanSize(s string) (int64, error) {
// Options carries the subset of command flags a job file may set. // Options carries the subset of command flags a job file may set.
type Options struct { type Options struct {
StrictConversion bool `yaml:"strict_conversion"`
LowercaseIdentifiers bool `yaml:"lowercase_identifiers"`
FlattenSchema bool `yaml:"flatten_schema"` FlattenSchema bool `yaml:"flatten_schema"`
Schema string `yaml:"schema"` Schema string `yaml:"schema"`
Package string `yaml:"package"` Package string `yaml:"package"`
@@ -285,6 +287,7 @@ type Options struct {
Types string `yaml:"types"` Types string `yaml:"types"`
ArrayNullable string `yaml:"array_nullable"` ArrayNullable string `yaml:"array_nullable"`
ContinueOnError bool `yaml:"continue_on_error"` ContinueOnError bool `yaml:"continue_on_error"`
AtomicExecution bool `yaml:"atomic_execution"`
// FullDDL makes direct pgsql output execute the full idempotent DDL instead of // FullDDL makes direct pgsql output execute the full idempotent DDL instead of
// diffing against the live database first. // diffing against the live database first.
FullDDL bool `yaml:"full_ddl"` FullDDL bool `yaml:"full_ddl"`
@@ -479,6 +482,14 @@ func (s *Set) Validate() error {
func (j *Job) validate() []string { func (j *Job) validate() []string {
var e []string var e []string
if j.Options.AtomicExecution {
if j.Options.ContinueOnError {
e = append(e, "atomic_execution cannot be combined with continue_on_error")
}
if (j.Command != CommandConvert && j.Command != CommandMerge) || j.Output == nil || j.Output.ConnEnv == "" || j.Output.Format != "pgsql" {
e = append(e, "atomic_execution requires convert/merge with pgsql database output")
}
}
switch j.Command { switch j.Command {
case CommandConvert, CommandMerge, CommandScriptsList, CommandScriptsExec, case CommandConvert, CommandMerge, CommandScriptsList, CommandScriptsExec,
+19 -19
View File
@@ -61,6 +61,7 @@ var MariaDBToCanonicalTypes = map[string]string{
// CanonicalToMariaDBTypes maps canonical types to MariaDB/MySQL types. // CanonicalToMariaDBTypes maps canonical types to MariaDB/MySQL types.
var CanonicalToMariaDBTypes = map[string]string{ var CanonicalToMariaDBTypes = map[string]string{
"bool": "TINYINT(1)", "bool": "TINYINT(1)",
"boolean": "TINYINT(1)",
"int8": "TINYINT", "int8": "TINYINT",
"int16": "SMALLINT", "int16": "SMALLINT",
"int": "INT", "int": "INT",
@@ -114,22 +115,28 @@ func NormalizeMariaDBType(baseType string) string {
// ConvertMariaDBToCanonical converts a MariaDB/MySQL type name to the canonical type. // ConvertMariaDBToCanonical converts a MariaDB/MySQL type name to the canonical type.
// Strips dimension parameters and normalizes aliases. Defaults to "string". // Strips dimension parameters and normalizes aliases. Defaults to "string".
func ConvertMariaDBToCanonical(mariadbType string) string { func ConvertMariaDBToCanonical(mariadbType string) string {
base := strings.ToLower(strings.TrimSpace(mariadbType)) lower := strings.ToLower(strings.TrimSpace(mariadbType))
if idx := strings.Index(base, "("); idx >= 0 { base := lower
base = strings.TrimSpace(base[:idx]) if idx := strings.IndexAny(base, "( "); idx >= 0 {
base = base[:idx]
}
canonical, ok := MariaDBToCanonicalTypes[base]
if ok {
if strings.Contains(lower, "unsigned") || strings.Contains(lower, "zerofill") {
switch canonical {
case "int8":
return "uint8"
case "int16":
return "uint16"
case "int":
return "uint"
case "int64":
return "uint64"
}
} }
if canonical, ok := MariaDBToCanonicalTypes[base]; ok {
return canonical return canonical
} }
// Prefix match for composite types (e.g., "unsigned bigint")
for key, canonical := range MariaDBToCanonicalTypes {
if strings.HasPrefix(base, key) {
return canonical
}
}
return "string" return "string"
} }
@@ -145,12 +152,5 @@ func ConvertCanonicalToMariaDB(canonicalType string) string {
return mariadbType return mariadbType
} }
// Prefix fallback
for canonical, mariadb := range CanonicalToMariaDBTypes {
if strings.HasPrefix(lower, canonical) {
return mariadb
}
}
return "VARCHAR(255)" return "VARCHAR(255)"
} }
+13
View File
@@ -0,0 +1,13 @@
package mariadb
import "testing"
func TestIntegerSignedness(t *testing.T) {
for _, tc := range []struct{ input, want string }{
{"tinyint(3) unsigned", "uint8"}, {"smallint unsigned", "uint16"}, {"mediumint unsigned", "uint"}, {"int unsigned", "uint"}, {"bigint unsigned", "uint64"}, {"bigint zerofill", "uint64"}, {"bigint", "int64"}, {"varchar(32)", "string"},
} {
if got := ConvertMariaDBToCanonical(tc.input); got != tc.want {
t.Errorf("%s: got %s want %s", tc.input, got, tc.want)
}
}
}
+11
View File
@@ -256,9 +256,19 @@ func (d *Column) SQLName() string {
return strings.ToLower(d.Name) return strings.ToLower(d.Name)
} }
// IndexPart is an ordered MySQL index key part. Name and Expression are mutually
// exclusive; expressions are executable SQL and are not identifier-normalized.
type IndexPart struct {
Name string `json:"name,omitempty" yaml:"name,omitempty" xml:"name,omitempty"`
Expression string `json:"expression,omitempty" yaml:"expression,omitempty" xml:"expression,omitempty"`
Length int `json:"length,omitempty" yaml:"length,omitempty" xml:"length,omitempty"`
Descending bool `json:"descending,omitempty" yaml:"descending,omitempty" xml:"descending,omitempty"`
}
// Index represents a database index for optimizing query performance. // Index represents a database index for optimizing query performance.
// Indexes can be unique, partial, or include additional columns. // Indexes can be unique, partial, or include additional columns.
type Index struct { type Index struct {
KeyParts []IndexPart `json:"key_parts,omitempty" yaml:"key_parts,omitempty" xml:"key_parts,omitempty"`
Name string `json:"name" yaml:"name" xml:"name"` Name string `json:"name" yaml:"name" xml:"name"`
Description string `json:"description,omitempty" yaml:"description,omitempty" xml:"description,omitempty"` Description string `json:"description,omitempty" yaml:"description,omitempty" xml:"description,omitempty"`
Table string `json:"table,omitempty" yaml:"table,omitempty" xml:"table,omitempty"` Table string `json:"table,omitempty" yaml:"table,omitempty" xml:"table,omitempty"`
@@ -318,6 +328,7 @@ func (d *Relationship) SQLName() string {
// Constraint represents a database constraint that enforces data integrity rules. // Constraint represents a database constraint that enforces data integrity rules.
// Constraints can be primary keys, foreign keys, unique constraints, check constraints, or not-null constraints. // Constraints can be primary keys, foreign keys, unique constraints, check constraints, or not-null constraints.
type Constraint struct { type Constraint struct {
Metadata map[string]any `json:"metadata,omitempty" yaml:"metadata,omitempty" xml:"-"`
Name string `json:"name" yaml:"name" xml:"name"` Name string `json:"name" yaml:"name" xml:"name"`
Type ConstraintType `json:"type" yaml:"type" xml:"type"` Type ConstraintType `json:"type" yaml:"type" xml:"type"`
Columns []string `json:"columns" yaml:"columns" xml:"columns"` Columns []string `json:"columns" yaml:"columns" xml:"columns"`
+194
View File
@@ -0,0 +1,194 @@
package mysql
import (
"regexp"
"strings"
)
// columnDefaults extracts native DEFAULT expressions from SHOW CREATE TABLE.
// MySQL can truncate binary defaults in information_schema (for example X'00FF'
// appears as just "0x"). SHOW CREATE is authoritative for their SQL spelling.
// Separators and whitespace inside quoted strings, identifiers and nested
// expressions are never treated as definition/default boundaries.
func columnDefaults(create string) map[string]string {
out := map[string]string{}
depth, start := 0, 0
var quoted byte
definitions := []string{}
for i := 0; i < len(create); i++ {
ch := create[i]
if quoted != 0 {
if ch == '\\' && quoted != '`' && i+1 < len(create) {
i++
continue
}
if ch == quoted {
if i+1 < len(create) && create[i+1] == quoted {
i++
} else {
quoted = 0
}
}
continue
}
if ch == '\'' || ch == '"' || ch == '`' {
quoted = ch
continue
}
switch ch {
case '(':
depth++
if depth == 1 {
start = i + 1
}
case ',':
if depth == 1 {
definitions = append(definitions, create[start:i])
start = i + 1
}
case ')':
depth--
if depth == 0 {
definitions = append(definitions, create[start:i])
}
}
if ch == ')' && depth == 0 {
break
}
}
for _, def := range definitions {
def = strings.TrimSpace(def)
if len(def) == 0 || def[0] != '`' {
continue
}
var name strings.Builder
end := 1
for end < len(def) {
if def[end] == '`' {
if end+1 < len(def) && def[end+1] == '`' {
name.WriteByte('`')
end += 2
continue
}
end++
break
}
name.WriteByte(def[end])
end++
}
// Consume full tokens so DEFAULT inside a generated expression or literal
// cannot masquerade as a column default.
for end < len(def) {
for end < len(def) && isSpace(def[end]) {
end++
}
begin := end
end = tokenEnd(def, end)
if end == begin {
break
}
if strings.EqualFold(def[begin:end], "DEFAULT") {
for end < len(def) && isSpace(def[end]) {
end++
}
finish := tokenEnd(def, end)
out[name.String()] = def[end:finish]
break
}
}
}
return out
}
func isSpace(b byte) bool { return b == ' ' || b == '\n' || b == '\r' || b == '\t' }
func tokenEnd(s string, start int) int {
depth := 0
var quoted byte
for i := start; i < len(s); i++ {
ch := s[i]
if quoted != 0 {
if ch == '\\' && quoted != '`' && i+1 < len(s) {
i++
continue
}
if ch == quoted {
if i+1 < len(s) && s[i+1] == quoted {
i++
} else {
quoted = 0
}
}
continue
}
if ch == '\'' || ch == '"' || ch == '`' {
quoted = ch
continue
}
if ch == '(' {
depth++
}
if ch == ')' {
depth--
}
if depth == 0 && isSpace(ch) {
return i
}
}
return len(s)
}
// SHOW CREATE has native options not exposed by STATISTICS/COLUMNS. Ignore
// quoted identifiers and values when detecting options outside our contract.
func unquotedSQL(create string) string {
var code strings.Builder
var quoted byte
for i := 0; i < len(create); i++ {
ch := create[i]
if quoted != 0 {
if ch == '\\' && quoted != '`' && i+1 < len(create) {
i++
continue
}
if ch == quoted {
if i+1 < len(create) && create[i+1] == quoted {
i++
} else {
quoted = 0
code.WriteByte(' ')
}
}
continue
}
if ch == '\'' || ch == '"' || ch == '`' {
quoted = ch
code.WriteByte(' ')
continue
}
code.WriteByte(ch)
}
return code.String()
}
var autoIncrementOption = regexp.MustCompile(`(?i)\bAUTO_INCREMENT\s*=\s*([0-9]+)`)
func autoIncrementValue(create string) string {
if match := autoIncrementOption.FindStringSubmatch(unquotedSQL(create)); len(match) > 1 {
return match[1]
}
return "1"
}
func unsupportedNativeOptions(create string) []string {
words := strings.FieldsFunc(strings.ToUpper(unquotedSQL(create)), func(r rune) bool { return (r < 'A' || r > 'Z') && r != '_' })
var out []string
for i, w := range words {
switch w {
case "ENGINE_ATTRIBUTE", "SECONDARY_ENGINE_ATTRIBUTE", "KEY_BLOCK_SIZE", "COLUMN_FORMAT", "STORAGE":
out = append(out, w)
case "PARSER":
if i > 0 && words[i-1] == "WITH" {
out = append(out, "WITH PARSER")
}
}
}
return out
}
+54
View File
@@ -0,0 +1,54 @@
package mysql
import "testing"
func TestUnescapeExpression(t *testing.T) {
for _, tc := range []struct{ input, want string }{
{`_utf8mb4\'one\\ntwo\'`, `_utf8mb4'one\ntwo'`},
{`concat(\'don\\\'t\',\' change\')`, `concat('don\'t',' change')`},
{`(uuid())`, `(uuid())`},
} {
if got := unescapeExpression(tc.input); got != tc.want {
t.Errorf("%q: got %q want %q", tc.input, got, tc.want)
}
}
}
func TestNativeColumnDefaults(t *testing.T) {
create := "CREATE TABLE `odd(table` (\n" +
" `a,b` varbinary(8) DEFAULT 0x0027FF,\n" +
" `tick``name` varchar(30) DEFAULT 'don''t, (split)',\n" +
" `expr` varchar(30) DEFAULT (concat('DEFAULT, ', (uuid()))),\n" +
" `generated` varchar(30) GENERATED ALWAYS AS (concat('DEFAULT',`expr`)) VIRTUAL,\n" +
" `new\nline` int DEFAULT 3,\n" +
" PRIMARY KEY (`a,b`)\n) ENGINE=InnoDB"
got := columnDefaults(create)
want := map[string]string{"a,b": "0x0027FF", "tick`name": "'don''t, (split)'", "expr": "(concat('DEFAULT, ', (uuid())))", "new\nline": "3"}
if len(got) != len(want) {
t.Fatalf("defaults: %v", got)
}
for name, value := range want {
if got[name] != value {
t.Errorf("%s: got %q want %q", name, got[name], value)
}
}
}
func TestNativeOptionsIgnoreQuotedValues(t *testing.T) {
create := "CREATE TABLE `WITH PARSER` (`STORAGE` int COMMENT 'KEY_BLOCK_SIZE', KEY `ENGINE_ATTRIBUTE` (`STORAGE`) WITH PARSER `ngram`) ENGINE=InnoDB"
options := unsupportedNativeOptions(create)
if len(options) != 1 || options[0] != "WITH PARSER" {
t.Fatalf("native options: %v", options)
}
}
func TestNativeAutoIncrementIgnoresQuotedValues(t *testing.T) {
for _, tc := range []struct{ create, want string }{
{"CREATE TABLE `x` (`id` bigint AUTO_INCREMENT) ENGINE=InnoDB AUTO_INCREMENT=18446744073709551615", "18446744073709551615"},
{"CREATE TABLE `x` (`id` bigint AUTO_INCREMENT) COMMENT='AUTO_INCREMENT=12'", "1"},
} {
if got := autoIncrementValue(tc.create); got != tc.want {
t.Errorf("got %s want %s", got, tc.want)
}
}
}
+252 -43
View File
@@ -4,9 +4,10 @@ import (
"context" "context"
"database/sql" "database/sql"
"fmt" "fmt"
"regexp"
"strings" "strings"
_ "github.com/go-sql-driver/mysql" mysqlDriver "github.com/go-sql-driver/mysql"
"git.warky.dev/wdevs/relspecgo/pkg/mariadb" "git.warky.dev/wdevs/relspecgo/pkg/mariadb"
"git.warky.dev/wdevs/relspecgo/pkg/models" "git.warky.dev/wdevs/relspecgo/pkg/models"
@@ -31,12 +32,13 @@ func (r *Reader) ReadDatabase() (*models.Database, error) {
return nil, fmt.Errorf("failed to connect: %w", err) return nil, fmt.Errorf("failed to connect: %w", err)
} }
defer r.close() defer r.close()
var name, version string var name, version, sqlMode string
if err := r.db.QueryRowContext(r.ctx, "SELECT DATABASE()").Scan(&name); err != nil { if err := r.db.QueryRowContext(r.ctx, "SELECT DATABASE(), @@SESSION.sql_mode").Scan(&name, &sqlMode); err != nil {
return nil, fmt.Errorf("failed to get database name: %w", err) return nil, fmt.Errorf("failed to get database name: %w", err)
} }
_ = r.db.QueryRowContext(r.ctx, "SELECT VERSION()").Scan(&version) _ = r.db.QueryRowContext(r.ctx, "SELECT VERSION()").Scan(&version)
db := models.InitDatabase(name) db := models.InitDatabase(name)
db.Metadata["mysql_sql_mode"] = sqlMode
db.DatabaseType, db.SourceFormat, db.DatabaseVersion = models.MySQLDatabaseType, "mysql", version db.DatabaseType, db.SourceFormat, db.DatabaseVersion = models.MySQLDatabaseType, "mysql", version
schemas, err := r.querySchemas(name) schemas, err := r.querySchemas(name)
if err != nil { if err != nil {
@@ -48,8 +50,10 @@ func (r *Reader) ReadDatabase() (*models.Database, error) {
return nil, err return nil, err
} }
schema.Tables = tables schema.Tables = tables
schema.Metadata["mysql_sql_mode"] = sqlMode
for _, table := range tables { for _, table := range tables {
table.Columns, err = r.queryColumns(schema.Name, table.Name) table.Metadata["mysql_sql_mode"] = sqlMode
table.Columns, err = r.queryColumns(schema.Name, table)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@@ -62,6 +66,15 @@ func (r *Reader) ReadDatabase() (*models.Database, error) {
return nil, err return nil, err
} }
table.RefSchema = schema table.RefSchema = schema
for _, c := range table.Constraints {
if c.Type == models.PrimaryKeyConstraint {
for _, n := range c.Columns {
if col := table.Columns[n]; col != nil {
col.IsPrimaryKey = true
}
}
}
}
for _, c := range table.Constraints { for _, c := range table.Constraints {
if c.Type == models.ForeignKeyConstraint { if c.Type == models.ForeignKeyConstraint {
r.deriveRelationship(table, c) r.deriveRelationship(table, c)
@@ -71,6 +84,9 @@ func (r *Reader) ReadDatabase() (*models.Database, error) {
schema.RefDatabase = db schema.RefDatabase = db
db.Schemas = append(db.Schemas, schema) db.Schemas = append(db.Schemas, schema)
} }
if err := r.readUnsupported(db, name); err != nil {
return nil, err
}
return db, nil return db, nil
} }
@@ -97,7 +113,17 @@ func (r *Reader) ReadTable() (*models.Table, error) {
} }
func (r *Reader) connect() error { func (r *Reader) connect() error {
db, err := sql.Open("mysql", r.options.ConnectionString) cfg, err := mysqlDriver.ParseDSN(r.options.ConnectionString)
if err != nil {
return err
}
if cfg.Params == nil {
cfg.Params = map[string]string{}
}
// SHOW CREATE parsing requires quoted column identifiers, including when the
// caller's metadata connection would otherwise disable show-create quoting.
cfg.Params["sql_quote_show_create"] = "1"
db, err := sql.Open("mysql", cfg.FormatDSN())
if err != nil { if err != nil {
return err return err
} }
@@ -134,61 +160,147 @@ func (r *Reader) querySchemas(current string) ([]*models.Schema, error) {
} }
func (r *Reader) queryTables(schema string) ([]*models.Table, error) { func (r *Reader) queryTables(schema string) ([]*models.Table, error) {
rows, err := r.db.QueryContext(r.ctx, "SELECT TABLE_NAME FROM information_schema.TABLES WHERE TABLE_SCHEMA = ? AND TABLE_TYPE = 'BASE TABLE' ORDER BY TABLE_NAME", schema) rows, err := r.db.QueryContext(r.ctx, "SELECT TABLE_NAME, ENGINE, TABLE_COLLATION, TABLE_COMMENT, CREATE_OPTIONS, AUTO_INCREMENT FROM information_schema.TABLES WHERE TABLE_SCHEMA = ? AND TABLE_TYPE = 'BASE TABLE' ORDER BY TABLE_NAME", schema)
if err != nil { if err != nil {
return nil, err return nil, err
} }
defer rows.Close() defer rows.Close()
var out []*models.Table var out []*models.Table
for rows.Next() { for rows.Next() {
var n string var n, engine, collation, comment, options string
if err := rows.Scan(&n); err != nil { var auto sql.NullString
if err := rows.Scan(&n, &engine, &collation, &comment, &options, &auto); err != nil {
return nil, err return nil, err
} }
out = append(out, models.InitTable(n, schema)) t := models.InitTable(n, schema)
t.Comment = comment
t.Metadata["mysql_engine"], t.Metadata["mysql_collation"], t.Metadata["mysql_create_options"] = engine, collation, options
if auto.Valid {
t.Metadata["mysql_auto_increment"] = auto.String
}
out = append(out, t)
} }
return out, rows.Err() return out, rows.Err()
} }
func (r *Reader) queryColumns(schema, table string) (map[string]*models.Column, error) { var onUpdateExtra = regexp.MustCompile(`on update current_timestamp(?:\(\d*\))?`)
rows, err := r.db.QueryContext(r.ctx, `SELECT COLUMN_NAME, COLUMN_TYPE, IS_NULLABLE, COLUMN_DEFAULT, ORDINAL_POSITION, EXTRA, COLUMN_COMMENT FROM information_schema.COLUMNS WHERE TABLE_SCHEMA = ? AND TABLE_NAME = ? ORDER BY ORDINAL_POSITION`, schema, table)
func (r *Reader) queryColumns(schema string, table *models.Table) (map[string]*models.Column, error) {
rows, err := r.db.QueryContext(r.ctx, `SELECT COLUMN_NAME, COLUMN_TYPE, IS_NULLABLE, COLUMN_DEFAULT, ORDINAL_POSITION, EXTRA, COLUMN_COMMENT,
CHARACTER_MAXIMUM_LENGTH, NUMERIC_PRECISION, NUMERIC_SCALE, DATETIME_PRECISION, COLLATION_NAME, CHARACTER_SET_NAME, GENERATION_EXPRESSION, SRS_ID
FROM information_schema.COLUMNS WHERE TABLE_SCHEMA=? AND TABLE_NAME=? ORDER BY ORDINAL_POSITION`, schema, table.Name)
if err != nil { if err != nil {
return nil, err return nil, fmt.Errorf("columns %s.%s: %w", schema, table.Name, err)
} }
defer rows.Close() defer rows.Close()
out := map[string]*models.Column{} out := map[string]*models.Column{}
for rows.Next() { for rows.Next() {
var name, typ, nullable, extra, comment string var name, typ, nullable, extra, comment string
var def sql.NullString var def, collation, charset, generation sql.NullString
var length, precision, scale, datetime, srid sql.NullInt64
var pos int var pos int
if err := rows.Scan(&name, &typ, &nullable, &def, &pos, &extra, &comment); err != nil { if err := rows.Scan(&name, &typ, &nullable, &def, &pos, &extra, &comment, &length, &precision, &scale, &datetime, &collation, &charset, &generation, &srid); err != nil {
return nil, err return nil, err
} }
c := models.InitColumn(name, table, schema) c := models.InitColumn(name, table.Name, schema)
c.Type = r.mapDataType(typ) c.Type, c.NotNull, c.Sequence, c.Comment = r.mapDataType(typ), nullable == "NO", uint(pos), comment
c.NotNull = strings.EqualFold(nullable, "NO") // The native declaration preserves mediumint, enum/set, binary/blob sizes,
c.Sequence = uint(pos) // unsigned/zerofill and timestamp versus datetime. Canonical dimensions remain editable.
c.Comment = comment c.Length, c.Precision, c.Scale = int(length.Int64), int(precision.Int64), int(scale.Int64)
if datetime.Valid {
c.Precision = int(datetime.Int64)
}
c.Collation = collation.String
c.Metadata["mysql_type"] = typ
c.Metadata["mysql_type_signature"] = typeSignature(c)
c.Metadata["mysql_charset"] = charset.String
c.Metadata["mysql_datetime_precision"] = datetime.Int64
c.Metadata["mysql_default_expression"] = strings.Contains(extra, "DEFAULT_GENERATED") || strings.HasPrefix(strings.ToLower(def.String), "current_timestamp") && (c.Type == "timestamp")
if def.Valid { if def.Valid {
c.Default = def.String c.Default = def.String
if strings.Contains(extra, "DEFAULT_GENERATED") {
c.Default = unescapeExpression(def.String)
}
if strings.HasPrefix(typ, "bit(") && def.Valid {
c.Metadata["mysql_default_expression"] = true
}
}
c.AutoIncrement = strings.Contains(extra, "auto_increment")
c.Generated, c.GenerationExpression = generation.String != "", unescapeExpression(generation.String)
if strings.Contains(extra, "STORED GENERATED") {
c.Metadata["mysql_generation_storage"] = "STORED"
} else if c.Generated {
c.Metadata["mysql_generation_storage"] = "VIRTUAL"
}
c.Metadata["mysql_invisible"] = strings.Contains(extra, "INVISIBLE")
remaining := onUpdateExtra.ReplaceAllString(strings.ToLower(extra), "")
for _, known := range []string{"auto_increment", "default_generated", "stored generated", "virtual generated", "invisible"} {
remaining = strings.ReplaceAll(remaining, known, "")
}
if remaining = strings.TrimSpace(remaining); remaining != "" {
c.Metadata["mysql_unsupported_extra"] = remaining
}
if i := strings.Index(strings.ToLower(extra), "on update "); i >= 0 {
c.Metadata["mysql_on_update"] = strings.Fields(extra[i+10:])[0]
}
if srid.Valid {
c.Metadata["mysql_srs_id"] = srid.Int64
} }
c.AutoIncrement = strings.Contains(strings.ToLower(extra), "auto_increment")
out[name] = c out[name] = c
} }
return out, rows.Err() if err := rows.Err(); err != nil {
return nil, err
}
var tableName, create string
qualified := "`" + strings.ReplaceAll(schema, "`", "``") + "`.`" + strings.ReplaceAll(table.Name, "`", "``") + "`"
if err := r.db.QueryRowContext(r.ctx, "SHOW CREATE TABLE "+qualified).Scan(&tableName, &create); err != nil {
return nil, fmt.Errorf("native defaults %s.%s: %w", schema, table.Name, err)
}
// information_schema AUTO_INCREMENT may be stale because of statistics caching.
for _, c := range out {
if c.AutoIncrement {
table.Metadata["mysql_auto_increment"] = autoIncrementValue(create)
break
}
}
if options := unsupportedNativeOptions(create); len(options) > 0 {
table.Metadata["mysql_unsupported_options"] = strings.Join(options, ", ")
}
for name, sql := range columnDefaults(create) {
c := out[name]
if c == nil || c.Default == nil {
continue
}
if c.Type == "bytea" {
c.Default = sql
}
c.Metadata["mysql_default_sql"] = sql
c.Metadata["mysql_default_value"] = fmt.Sprint(c.Default)
}
return out, nil
}
func typeSignature(c *models.Column) string {
return fmt.Sprintf("%s/%d/%d/%d", c.Type, c.Length, c.Precision, c.Scale)
} }
func (r *Reader) queryConstraints(schema, table string) (map[string]*models.Constraint, error) { func (r *Reader) queryConstraints(schema, table string) (map[string]*models.Constraint, error) {
rows, err := r.db.QueryContext(r.ctx, `SELECT CONSTRAINT_NAME, CONSTRAINT_TYPE, COLUMN_NAME, REFERENCED_TABLE_SCHEMA, REFERENCED_TABLE_NAME, REFERENCED_COLUMN_NAME, ORDINAL_POSITION FROM information_schema.KEY_COLUMN_USAGE k JOIN information_schema.TABLE_CONSTRAINTS t USING (CONSTRAINT_SCHEMA, TABLE_NAME, CONSTRAINT_NAME) WHERE k.TABLE_SCHEMA=? AND k.TABLE_NAME=? ORDER BY CONSTRAINT_NAME, ORDINAL_POSITION`, schema, table) rows, err := r.db.QueryContext(r.ctx, `SELECT k.CONSTRAINT_NAME, t.CONSTRAINT_TYPE, k.COLUMN_NAME,
k.REFERENCED_TABLE_SCHEMA, k.REFERENCED_TABLE_NAME, k.REFERENCED_COLUMN_NAME, k.ORDINAL_POSITION,
rc.DELETE_RULE, rc.UPDATE_RULE FROM information_schema.KEY_COLUMN_USAGE k
JOIN information_schema.TABLE_CONSTRAINTS t USING (CONSTRAINT_SCHEMA,TABLE_NAME,CONSTRAINT_NAME)
LEFT JOIN information_schema.REFERENTIAL_CONSTRAINTS rc ON rc.CONSTRAINT_SCHEMA=k.CONSTRAINT_SCHEMA AND rc.TABLE_NAME=k.TABLE_NAME AND rc.CONSTRAINT_NAME=k.CONSTRAINT_NAME
WHERE k.TABLE_SCHEMA=? AND k.TABLE_NAME=? ORDER BY k.CONSTRAINT_NAME,k.ORDINAL_POSITION`, schema, table)
if err != nil { if err != nil {
return nil, err return nil, fmt.Errorf("constraints %s.%s: %w", schema, table, err)
} }
defer rows.Close()
out := map[string]*models.Constraint{} out := map[string]*models.Constraint{}
for rows.Next() { for rows.Next() {
var name, typ, col, rs, rt, rc string var name, typ string
var col, rs, rt, rc, del, upd sql.NullString
var pos int var pos int
if err := rows.Scan(&name, &typ, &col, &rs, &rt, &rc, &pos); err != nil { if err := rows.Scan(&name, &typ, &col, &rs, &rt, &rc, &pos, &del, &upd); err != nil {
rows.Close()
return nil, err return nil, err
} }
c := out[name] c := out[name]
@@ -201,31 +313,54 @@ func (r *Reader) queryConstraints(schema, table string) (map[string]*models.Cons
ct = models.ForeignKeyConstraint ct = models.ForeignKeyConstraint
} }
c = models.InitConstraint(name, ct) c = models.InitConstraint(name, ct)
c.Schema = schema c.Schema, c.Table = schema, table
c.Table = table c.ReferencedSchema, c.ReferencedTable = rs.String, rt.String
c.ReferencedSchema = rs c.OnDelete, c.OnUpdate = del.String, upd.String
c.ReferencedTable = rt
out[name] = c out[name] = c
} }
c.Columns = append(c.Columns, col) if col.Valid {
if rc != "" { c.Columns = append(c.Columns, col.String)
c.ReferencedColumns = append(c.ReferencedColumns, rc)
} }
if rc.Valid {
c.ReferencedColumns = append(c.ReferencedColumns, rc.String)
}
}
err = rows.Err()
rows.Close()
if err != nil {
return nil, err
}
rows, err = r.db.QueryContext(r.ctx, `SELECT t.CONSTRAINT_NAME,c.CHECK_CLAUSE,t.ENFORCED FROM information_schema.TABLE_CONSTRAINTS t JOIN information_schema.CHECK_CONSTRAINTS c USING (CONSTRAINT_SCHEMA,CONSTRAINT_NAME) WHERE t.TABLE_SCHEMA=? AND t.TABLE_NAME=? AND t.CONSTRAINT_TYPE='CHECK' ORDER BY t.CONSTRAINT_NAME`, schema, table)
if err != nil {
return nil, fmt.Errorf("checks %s.%s (requires MySQL 8.0.16+): %w", schema, table, err)
}
defer rows.Close()
for rows.Next() {
var name, expr, enforced string
if err := rows.Scan(&name, &expr, &enforced); err != nil {
return nil, err
}
c := models.InitConstraint(name, models.CheckConstraint)
c.Schema, c.Table, c.Expression = schema, table, unescapeExpression(expr)
c.Metadata = map[string]any{"mysql_enforced": enforced == "YES"}
out[name] = c
} }
return out, rows.Err() return out, rows.Err()
} }
func (r *Reader) queryIndexes(schema, table string) (map[string]*models.Index, error) { func (r *Reader) queryIndexes(schema, table string) (map[string]*models.Index, error) {
rows, err := r.db.QueryContext(r.ctx, `SELECT INDEX_NAME, NON_UNIQUE, COLUMN_NAME, SEQ_IN_INDEX, INDEX_TYPE FROM information_schema.STATISTICS WHERE TABLE_SCHEMA=? AND TABLE_NAME=? AND INDEX_NAME <> 'PRIMARY' ORDER BY INDEX_NAME, SEQ_IN_INDEX`, schema, table) rows, err := r.db.QueryContext(r.ctx, `SELECT INDEX_NAME, NON_UNIQUE, COLUMN_NAME, SEQ_IN_INDEX, INDEX_TYPE, EXPRESSION, SUB_PART, COLLATION, IS_VISIBLE, INDEX_COMMENT FROM information_schema.STATISTICS WHERE TABLE_SCHEMA=? AND TABLE_NAME=? ORDER BY INDEX_NAME,SEQ_IN_INDEX`, schema, table)
if err != nil { if err != nil {
return nil, err return nil, fmt.Errorf("indexes %s.%s (requires MySQL 8.0.13+): %w", schema, table, err)
} }
defer rows.Close() defer rows.Close()
out := map[string]*models.Index{} out := map[string]*models.Index{}
for rows.Next() { for rows.Next() {
var name, col, typ string var name, typ, visible, comment string
var col, expr, direction sql.NullString
var prefix sql.NullInt64
var non, seq int var non, seq int
if err := rows.Scan(&name, &non, &col, &seq, &typ); err != nil { if err := rows.Scan(&name, &non, &col, &seq, &typ, &expr, &prefix, &direction, &visible, &comment); err != nil {
return nil, err return nil, err
} }
i := out[name] i := out[name]
@@ -233,22 +368,96 @@ func (r *Reader) queryIndexes(schema, table string) (map[string]*models.Index, e
i = models.InitIndex(name, table, schema) i = models.InitIndex(name, table, schema)
i.Unique = non == 0 i.Unique = non == 0
i.Type = strings.ToLower(typ) i.Type = strings.ToLower(typ)
i.Comment = comment
i.Metadata["mysql_visible"] = visible == "YES"
out[name] = i out[name] = i
} }
i.Columns = append(i.Columns, col) i.Columns = append(i.Columns, col.String)
i.KeyParts = append(i.KeyParts, models.IndexPart{Name: col.String, Expression: unescapeExpression(expr.String), Length: int(prefix.Int64), Descending: direction.String == "D"})
} }
return out, rows.Err() return out, rows.Err()
} }
// Record objects outside the supported table-schema contract rather than silently
// discarding them. Writers can warn or reject the conversion before any output.
func (r *Reader) readUnsupported(db *models.Database, schema string) error {
queries := []struct{ kind, sql string }{
{"view", `SELECT TABLE_NAME FROM information_schema.VIEWS WHERE TABLE_SCHEMA=?`},
{"trigger", `SELECT TRIGGER_NAME FROM information_schema.TRIGGERS WHERE TRIGGER_SCHEMA=?`},
{"routine", `SELECT ROUTINE_NAME FROM information_schema.ROUTINES WHERE ROUTINE_SCHEMA=?`},
{"event", `SELECT EVENT_NAME FROM information_schema.EVENTS WHERE EVENT_SCHEMA=?`},
{"partitioned table", `SELECT DISTINCT TABLE_NAME FROM information_schema.PARTITIONS WHERE TABLE_SCHEMA=? AND PARTITION_NAME IS NOT NULL`},
}
var unsupported []string
for _, q := range queries {
rows, err := r.db.QueryContext(r.ctx, q.sql, schema)
if err != nil {
return fmt.Errorf("discover %s: %w", q.kind, err)
}
for rows.Next() {
var name string
if err := rows.Scan(&name); err != nil {
rows.Close()
return err
}
unsupported = append(unsupported, q.kind+" "+schema+"."+name)
}
err = rows.Err()
rows.Close()
if err != nil {
return err
}
}
if len(unsupported) > 0 {
db.Metadata["mysql_unsupported"] = unsupported
for _, s := range db.Schemas {
s.Metadata["mysql_unsupported"] = unsupported
}
if r.options.Progress != nil {
for _, item := range unsupported {
r.options.Progress("warning: unsupported MySQL " + item)
}
}
}
return nil
}
func (r *Reader) deriveRelationship(t *models.Table, c *models.Constraint) { func (r *Reader) deriveRelationship(t *models.Table, c *models.Constraint) {
n := fmt.Sprintf("%s_to_%s", t.Name, c.ReferencedTable) n := fmt.Sprintf("%s_%s", t.Name, c.Name)
rel := models.InitRelationship(n, models.OneToMany) rel := models.InitRelationship(n, models.OneToMany)
rel.FromTable = t.Name rel.FromTable, rel.FromSchema = t.Name, t.Schema
rel.FromSchema = t.Schema
rel.FromColumns = append([]string(nil), c.Columns...) rel.FromColumns = append([]string(nil), c.Columns...)
rel.ToTable = c.ReferencedTable rel.ToTable, rel.ToSchema = c.ReferencedTable, c.ReferencedSchema
rel.ToSchema = c.ReferencedSchema
rel.ToColumns = append([]string(nil), c.ReferencedColumns...) rel.ToColumns = append([]string(nil), c.ReferencedColumns...)
rel.ForeignKey = c.Name rel.ForeignKey = c.Name
t.Relationships[n] = rel t.Relationships[n] = rel
} }
// MySQL information_schema escapes SQL expressions by an additional layer
// (including quote delimiters). This covers defaults, generated columns, checks,
// and functional indexes. Literal defaults are already values and bypass this.
func unescapeExpression(s string) string {
var out strings.Builder
for i := 0; i < len(s); i++ {
if s[i] == '\\' && i+1 < len(s) {
i++
switch s[i] {
case 'n':
out.WriteByte('\n')
case 'r':
out.WriteByte('\r')
case 't':
out.WriteByte('\t')
case '0':
out.WriteByte(0)
case 'Z':
out.WriteByte(26)
default:
out.WriteByte(s[i])
}
} else {
out.WriteByte(s[i])
}
}
return out.String()
}
+16 -1
View File
@@ -3,12 +3,13 @@ package mysql
import ( import (
"testing" "testing"
"git.warky.dev/wdevs/relspecgo/pkg/models"
"git.warky.dev/wdevs/relspecgo/pkg/readers" "git.warky.dev/wdevs/relspecgo/pkg/readers"
) )
func TestReaderMapDataType(t *testing.T) { func TestReaderMapDataType(t *testing.T) {
r := NewReader(&readers.ReaderOptions{}) r := NewReader(&readers.ReaderOptions{})
for _, tc := range []struct{ input, want string }{{"varchar(64)", "string"}, {"bigint unsigned", "int64"}, {"datetime", "timestamp"}, {"json", "json"}} { for _, tc := range []struct{ input, want string }{{"varchar(64)", "string"}, {"bigint unsigned", "uint64"}, {"datetime", "timestamp"}, {"json", "json"}} {
if got := r.mapDataType(tc.input); got != tc.want { if got := r.mapDataType(tc.input); got != tc.want {
t.Errorf("mapDataType(%q) = %q, want %q", tc.input, got, tc.want) t.Errorf("mapDataType(%q) = %q, want %q", tc.input, got, tc.want)
} }
@@ -20,3 +21,17 @@ func TestReaderRequiresConnectionString(t *testing.T) {
t.Fatal("expected missing connection string error") t.Fatal("expected missing connection string error")
} }
} }
func TestRelationshipNamesCannotCollapseDistinctForeignKeys(t *testing.T) {
table := models.InitTable("t", "app")
reader := NewReader(&readers.ReaderOptions{})
for _, c := range []*models.Constraint{
{Name: "a_to_b", ReferencedTable: "c"},
{Name: "a", ReferencedTable: "b_to_c"},
} {
reader.deriveRelationship(table, c)
}
if len(table.Relationships) != 2 {
t.Fatalf("distinct foreign keys collapsed: %+v", table.Relationships)
}
}
+1 -1
View File
@@ -152,7 +152,7 @@ func (r *Reader) readScripts() ([]*models.Script, error) {
if err != nil { if err != nil {
return fmt.Errorf("failed to read file %s: %w", path, err) return fmt.Errorf("failed to read file %s: %w", path, err)
} }
sql, err := assetloader.ProcessEmbedDirectives(path, string(content)) sql, err := assetloader.ProcessEmbedDirectivesWithOptions(path, string(content), assetloader.EmbedOptions{AssetRoot: r.options.FilePath})
if err != nil { if err != nil {
return err return err
} }
+136
View File
@@ -0,0 +1,136 @@
// Package safefile provides portable path validation and root-scoped file access.
package safefile
import (
"fmt"
"io"
"os"
"path/filepath"
"strings"
"unicode"
)
// Relative validates artifact paths consistently on Unix and Windows. It rejects
// traversal rather than normalizing it, so aliases cannot hide filename collisions.
func Relative(name string) (string, error) {
if name == "" || !filepath.IsLocal(name) || strings.ContainsAny(name, `\:<>"|?*`) {
return "", fmt.Errorf("unsafe relative path %q", name)
}
for _, r := range name {
if unicode.IsControl(r) {
return "", fmt.Errorf("control character in path %q", name)
}
}
for _, part := range strings.Split(filepath.ToSlash(name), "/") {
if part == "" || part == "." || part == ".." || strings.HasSuffix(part, ".") || strings.HasSuffix(part, " ") {
return "", fmt.Errorf("unsafe path component in %q", name)
}
base := strings.ToUpper(strings.SplitN(part, ".", 2)[0])
device := base == "CON" || base == "PRN" || base == "AUX" || base == "NUL" || base == "CONIN$" || base == "CONOUT$"
for _, prefix := range []string{"COM", "LPT"} {
if strings.HasPrefix(base, prefix) {
suffix := []rune(strings.TrimPrefix(base, prefix))
device = device || (len(suffix) == 1 && strings.ContainsRune("123456789¹²³", suffix[0]))
}
}
if device {
return "", fmt.Errorf("reserved device path %q", name)
}
}
return filepath.FromSlash(name), nil
}
// Check rejects existing symlinks and special files; os.Root separately enforces
// containment at open/rename time, including if a symlink changes after this check.
func Check(root *os.Root, name string) error {
if _, err := Relative(filepath.ToSlash(name)); err != nil {
return err
}
parts := strings.Split(name, string(filepath.Separator))
current := ""
for i, part := range parts {
current = filepath.Join(current, part)
info, err := root.Lstat(current)
if os.IsNotExist(err) {
return nil
}
if err != nil {
return err
}
if info.Mode()&os.ModeSymlink != 0 {
return fmt.Errorf("symlink is not allowed in path %q", name)
}
if i < len(parts)-1 && !info.IsDir() {
return fmt.Errorf("parent is not a directory in path %q", name)
}
if i == len(parts)-1 && !info.Mode().IsRegular() {
return fmt.Errorf("path is not a regular file: %q", name)
}
}
return nil
}
// Read opens through a pinned root and bounds both the declared and actual size.
func Read(root *os.Root, name string, maxBytes int64) ([]byte, error) {
if maxBytes <= 0 || maxBytes == (1<<63)-1 {
return nil, fmt.Errorf("file size limit must be positive and bounded")
}
if err := Check(root, name); err != nil {
return nil, err
}
file, err := root.Open(name)
if err != nil {
return nil, err
}
defer file.Close()
info, err := file.Stat()
if err != nil {
return nil, err
}
if !info.Mode().IsRegular() {
return nil, fmt.Errorf("asset is not a regular file: %q", name)
}
if info.Size() > maxBytes {
return nil, fmt.Errorf("asset %q exceeds %d byte limit", name, maxBytes)
}
data, err := io.ReadAll(io.LimitReader(file, maxBytes+1))
if err != nil {
return nil, err
}
if int64(len(data)) > maxBytes {
return nil, fmt.Errorf("asset %q exceeds %d byte limit", name, maxBytes)
}
return data, nil
}
// CheckRoot rejects symlinks in the configured root or its existing ancestors.
// The configured root is a trusted caller-selected boundary; OpenRoot pins it for
// subsequent operations. Missing components are allowed for output root creation.
func CheckRoot(path string) error {
absolute, err := filepath.Abs(path)
if err != nil {
return err
}
volume := filepath.VolumeName(absolute)
current := volume + string(filepath.Separator)
for _, part := range strings.Split(strings.TrimPrefix(absolute, current), string(filepath.Separator)) {
if part == "" {
continue
}
current = filepath.Join(current, part)
info, err := os.Lstat(current)
if os.IsNotExist(err) {
return nil
}
if err != nil {
return err
}
if info.Mode()&os.ModeSymlink != 0 {
return fmt.Errorf("symlink is not allowed in root path %q", path)
}
if !info.IsDir() {
return fmt.Errorf("root parent is not a directory: %q", current)
}
}
return nil
}
+101
View File
@@ -0,0 +1,101 @@
// Package sqllex recognizes PostgreSQL lexical boundaries without parsing SQL.
package sqllex
import "strings"
func start(c byte) bool { return c >= 128 || c == '_' || c >= 'a' && c <= 'z' || c >= 'A' && c <= 'Z' }
func ident(c byte) bool { return start(c) || c >= '0' && c <= '9' || c >= 128 }
// Walk visits named parameters and standalone line comments in executable SQL. Quoted
// strings/identifiers, dollar bodies, casts and nested block comments are opaque.
// Returning replacement text is literal: it is never scanned again.
func Walk(sql string, parameter, lineComment func(string) string) string {
var out strings.Builder
for i := 0; i < len(sql); {
begin := i
switch {
case strings.HasPrefix(sql[i:], "--"):
i += 2
for i < len(sql) && sql[i] != '\n' && sql[i] != '\r' {
i++
}
text := sql[begin:i]
if lineComment != nil && strings.TrimSpace(sql[strings.LastIndexAny(sql[:begin], "\r\n")+1:begin]) == "" {
text = lineComment(text)
}
out.WriteString(text)
continue
case strings.HasPrefix(sql[i:], "/*"):
i += 2
depth := 1
for i < len(sql) && depth > 0 {
if strings.HasPrefix(sql[i:], "/*") {
depth++
i += 2
} else if strings.HasPrefix(sql[i:], "*/") {
depth--
i += 2
} else {
i++
}
}
case sql[i] == '\'' || sql[i] == '"':
q := sql[i]
escapes := q == '\'' && i > 0 && (sql[i-1] == 'E' || sql[i-1] == 'e') && (i == 1 || !ident(sql[i-2]))
i++
for i < len(sql) {
if escapes && sql[i] == '\\' {
i++
if i < len(sql) {
i++
}
continue
}
if sql[i] == q {
i++
if i < len(sql) && sql[i] == q {
i++
continue
}
break
}
i++
}
case sql[i] == '$' && (i == 0 || !ident(sql[i-1]) && sql[i-1] != '$'):
j := i + 1
if j < len(sql) && start(sql[j]) {
for j < len(sql) && ident(sql[j]) {
j++
}
}
if j < len(sql) && sql[j] == '$' {
tag := sql[i : j+1]
end := strings.Index(sql[j+1:], tag)
if end < 0 {
i = len(sql)
} else {
i = j + 1 + end + len(tag)
}
} else {
i++
}
case strings.HasPrefix(sql[i:], "::"):
i += 2
case sql[i] == ':' && i+1 < len(sql) && start(sql[i+1]) && (i == 0 || !ident(sql[i-1]) && sql[i-1] != ':'):
i += 2
for i < len(sql) && ident(sql[i]) {
i++
}
text := sql[begin:i]
if parameter != nil {
text = parameter(text)
}
out.WriteString(text)
continue
default:
i++
}
out.WriteString(sql[begin:i])
}
return out.String()
}
+28
View File
@@ -0,0 +1,28 @@
package sqllex
import (
"strings"
"testing"
)
func TestWalkSQLContexts(t *testing.T) {
sql := `SELECT :x,:x, :x::text, ':hidden', 'it'':hidden', E'it\':hidden', ":hidden", $$:hidden$$, $名$:hidden$名$;
-- :hidden
/* :hidden /* :hidden */ :hidden */ SELECT :y, value::integer, x := 1;`
var names []string
got := Walk(sql, func(name string) string { names = append(names, name); return "$1 ${2} :inserted" }, nil)
if strings.Join(names, ",") != ":x,:x,:x,:y" {
t.Fatalf("parameters: %v", names)
}
if strings.Count(got, "$1 ${2} :inserted") != 4 || !strings.Contains(got, "value::integer") {
t.Fatalf("result: %s", got)
}
}
func TestUnterminatedSQLContextsAreOpaque(t *testing.T) {
for _, sql := range []string{"SELECT ':x", `SELECT ":x`, "DO $tag$:x", "/* :x", "-- :x"} {
if got := Walk(sql, func(string) string { t.Fatal("unexpected parameter"); return "" }, nil); got != sql {
t.Fatalf("changed %q", sql)
}
}
}
+157
View File
@@ -0,0 +1,157 @@
package transform
import (
"fmt"
"reflect"
"strings"
"git.warky.dev/wdevs/relspecgo/pkg/models"
)
// LowercaseIdentifiers copies the model and lowercases names and structured
// references. SQL expressions, defaults, enum values, comments and metadata are
// preserved. Callers must adjust identifiers inside their own SQL expressions.
// Collisions are rejected before any output is written.
func LowercaseIdentifiers(db *models.Database) (*models.Database, error) {
if db == nil {
return nil, fmt.Errorf("database is required")
}
seen := map[any]reflect.Value{}
nameFields := map[string]bool{}
for _, name := range strings.Fields("Name Schema Table Tablespace Collation TableName SchemaName FromTable FromSchema ToTable ToSchema ThroughTable ThroughSchema ForeignKey ReferencedTable ReferencedSchema OwnedByTable OwnedByColumn") {
nameFields[name] = true
}
listFields := map[string]bool{"Columns": true, "Include": true, "ReferencedColumns": true, "FromColumns": true, "ToColumns": true, "RunAfter": true}
var copyValue func(reflect.Value) (reflect.Value, error)
copyValue = func(v reflect.Value) (reflect.Value, error) {
if !v.IsValid() {
return v, nil
}
switch v.Kind() {
case reflect.Pointer:
if v.IsNil() || v.Type().Elem().PkgPath() != reflect.TypeOf(models.Database{}).PkgPath() {
return v, nil
}
if prior, ok := seen[v.Interface()]; ok {
return prior, nil
}
out := reflect.New(v.Type().Elem())
seen[v.Interface()] = out
copied, err := copyValue(v.Elem())
if err != nil {
return reflect.Value{}, err
}
out.Elem().Set(copied)
return out, nil
case reflect.Struct:
out := reflect.New(v.Type()).Elem()
out.Set(v)
for i := 0; i < v.NumField(); i++ {
field := v.Type().Field(i).Name
value := v.Field(i)
if nameFields[field] && value.Kind() == reflect.String {
out.Field(i).SetString(strings.ToLower(value.String()))
continue
}
if (v.Type() != reflect.TypeOf(models.Index{}) || field != "Columns") && listFields[field] && value.Kind() == reflect.Slice && value.Type().Elem().Kind() == reflect.String {
list := reflect.MakeSlice(value.Type(), value.Len(), value.Len())
for j := 0; j < value.Len(); j++ {
list.Index(j).SetString(strings.ToLower(value.Index(j).String()))
}
out.Field(i).Set(list)
continue
}
if field == "Metadata" || field == "Default" || field == "Properties" || field == "Permissions" {
continue
}
copied, err := copyValue(value)
if err != nil {
return reflect.Value{}, err
}
out.Field(i).Set(copied)
}
if v.Type() == reflect.TypeOf(models.Table{}) {
original := v.Interface().(models.Table)
copied := out.Addr().Interface().(*models.Table)
for originalKey, index := range original.Indexes {
if index == nil {
continue
}
target := copied.Indexes[strings.ToLower(originalKey)]
target.Columns = append([]string(nil), index.Columns...)
for i, reference := range index.Columns {
for _, column := range original.Columns {
if column != nil && (reference == column.Name || reference == `"`+strings.ReplaceAll(column.Name, `"`, `""`)+`"`) {
target.Columns[i] = strings.ToLower(column.Name)
break
}
}
}
}
}
return out, nil
case reflect.Slice:
if v.IsNil() {
return v, nil
}
out := reflect.MakeSlice(v.Type(), v.Len(), v.Len())
names := map[string]bool{}
for i := 0; i < v.Len(); i++ {
copied, err := copyValue(v.Index(i))
if err != nil {
return reflect.Value{}, err
}
out.Index(i).Set(copied)
if copied.Kind() == reflect.Pointer && !copied.IsNil() && copied.Elem().Kind() == reflect.Struct {
name := copied.Elem().FieldByName("Name")
if name.IsValid() {
n := name.String()
if names[n] {
return reflect.Value{}, fmt.Errorf("lowercase identifier collision: %q", n)
}
names[n] = true
}
}
}
return out, nil
case reflect.Map:
if v.IsNil() || v.Type().Elem().Kind() != reflect.Pointer {
return v, nil
}
out := reflect.MakeMap(v.Type())
names := map[string]bool{}
iter := v.MapRange()
for iter.Next() {
key := iter.Key()
if key.Kind() == reflect.String {
key = reflect.ValueOf(strings.ToLower(key.String())).Convert(key.Type())
}
if out.MapIndex(key).IsValid() {
return reflect.Value{}, fmt.Errorf("lowercase identifier collision: %v", key.Interface())
}
copied, err := copyValue(iter.Value())
if err != nil {
return reflect.Value{}, err
}
if copied.Kind() == reflect.Pointer && !copied.IsNil() && copied.Elem().Kind() == reflect.Struct {
name := copied.Elem().FieldByName("Name")
if name.IsValid() {
n := name.String()
if names[n] {
return reflect.Value{}, fmt.Errorf("lowercase identifier collision: %q", n)
}
names[n] = true
}
}
out.SetMapIndex(key, copied)
}
return out, nil
}
return v, nil
}
value, err := copyValue(reflect.ValueOf(db))
if err != nil {
return nil, err
}
return value.Interface().(*models.Database), nil
}
+53
View File
@@ -0,0 +1,53 @@
package transform
import (
"testing"
"git.warky.dev/wdevs/relspecgo/pkg/models"
)
func TestLowercaseIdentifiersReferencesAndData(t *testing.T) {
db := models.InitDatabase("DB")
s := models.InitSchema("App")
tbl := models.InitTable("Users", "App")
col := models.InitColumn("ID", "Users", "App")
tbl.Tablespace = "FastStore"
col.Collation = "MyCollation"
col.Default = "Keep CASE"
col.GenerationExpression = `"ID" + 1`
tbl.Columns[col.Name] = col
tbl.Constraints["FK"] = &models.Constraint{Name: "FK", Columns: []string{"ID"}, ReferencedSchema: "Other", ReferencedTable: "Target", ReferencedColumns: []string{"Key"}}
tbl.Indexes["IDX"] = &models.Index{Name: "IDX", Columns: []string{"ID", "lower('VALUE')"}, Include: []string{"ID"}, Where: `"ID" > 0`}
tbl.RefSchema = s
s.Tables = append(s.Tables, tbl)
db.Schemas = append(db.Schemas, s)
got, err := LowercaseIdentifiers(db)
if err != nil {
t.Fatal(err)
}
table := got.Schemas[0].Tables[0]
if table.Tablespace != "faststore" || table.Columns["id"].Collation != "mycollation" || got.Name != "db" || table.Name != "users" || table.Schema != "app" || table.Columns["id"].Name != "id" || table.Constraints["fk"].ReferencedColumns[0] != "key" || table.Indexes["idx"].Include[0] != "id" || table.RefSchema != got.Schemas[0] {
t.Fatalf("bad normalized model: %+v", table)
}
if table.Columns["id"].Default != "Keep CASE" || table.Columns["id"].GenerationExpression != `"ID" + 1` || table.Indexes["idx"].Where != `"ID" > 0` || table.Indexes["idx"].Columns[1] != "lower('VALUE')" {
t.Fatal("data/expression changed")
}
if tbl.Name != "Users" || tbl.Columns["ID"] != col {
t.Fatal("input mutated")
}
}
func TestLowercaseIdentifierCollisions(t *testing.T) {
db := models.InitDatabase("db")
s := models.InitSchema("app")
db.Schemas = append(db.Schemas, s)
s.Tables = []*models.Table{models.InitTable("Users", "app"), models.InitTable("users", "app")}
if _, err := LowercaseIdentifiers(db); err == nil {
t.Fatal("expected table collision")
}
s.Tables = s.Tables[:1]
s.Tables[0].Columns = map[string]*models.Column{"ID": {Name: "ID"}, "id": {Name: "id"}}
if _, err := LowercaseIdentifiers(db); err == nil {
t.Fatal("expected column collision")
}
}
+239
View File
@@ -0,0 +1,239 @@
package writers
import (
"context"
"encoding/json"
"errors"
"fmt"
"os"
"strings"
"time"
)
// ExecutionReport describes an entire invocation, including operations never attempted.
// ExecutedStatements counts operations still applied, excluding confirmed rollbacks.
type ExecutionReport struct {
Dialect string `json:"dialect,omitempty"`
TotalStatements int `json:"total_statements"`
ExecutedStatements int `json:"executed_statements"`
FailedStatements int `json:"failed_statements"`
SkippedStatements int `json:"skipped_statements"`
RolledBackStatements int `json:"rolled_back_statements"`
Outcome string `json:"outcome"`
Operations []ExecutionOperation `json:"operations"`
Schemas []SchemaReport `json:"schemas"`
Errors []ExecutionError `json:"errors,omitempty"`
StartTime string `json:"start_time"`
EndTime string `json:"end_time"`
}
type SchemaReport struct {
Name string `json:"name"`
Tables []TableReport `json:"tables"`
}
type TableReport struct {
Name string `json:"name"`
Created bool `json:"created"`
Error string `json:"error,omitempty"`
}
type ExecutionError struct {
StatementNumber int `json:"statement_number"`
Statement string `json:"statement"`
Error string `json:"error"`
}
// ExecutionOperation is one generated statement or supplied script, in execution order.
// A failed raw script may have committed some of its own statements; see Outcome.
type ExecutionOperation struct {
StatementNumber int `json:"statement_number"`
Name string `json:"name,omitempty"`
Statement string `json:"statement"`
Status string `json:"status"` // applied, executed (raw script), failed, skipped, rolled_back, unknown
Error string `json:"error,omitempty"`
Skip bool `json:"-"`
StopOnFailure bool `json:"-"`
}
// ExecutionFailure is returned even when continuation was explicitly requested.
type ExecutionFailure struct{ Report *ExecutionReport }
func (e *ExecutionFailure) Error() string {
message := fmt.Sprintf("%s execution failed: %d failed, %d executed, %d skipped, %d rolled back",
e.Report.Dialect, e.Report.FailedStatements, e.Report.ExecutedStatements,
e.Report.SkippedStatements, e.Report.RolledBackStatements)
if len(e.Report.Errors) > 0 {
first := e.Report.Errors[0]
message += fmt.Sprintf("; statement %d: %s", first.StatementNumber, first.Error)
}
return message
}
// ExecutionBackend operates on one connection. Begin switches Exec to a transaction.
// Atomic execution stops at the first failure and rolls back the whole invocation.
type ExecutionBackend struct {
Exec func(context.Context, int, string) error
Begin func(context.Context) error
Commit func(context.Context) error
Rollback func(context.Context) error
// ScriptManaged means arbitrary scripts can commit/rollback internally.
ScriptManaged bool
}
func (o *WriterOptions) ShouldContinue() bool {
legacy, _ := o.Metadata["ignore_errors"].(bool)
return o.ContinueOnError || legacy
}
// RunExecution applies a plan, records every outcome, and publishes metadata and reports.
func RunExecution(ctx context.Context, opts *WriterOptions, report *ExecutionReport, operations []ExecutionOperation, backend ExecutionBackend) error {
report.TotalStatements = len(operations)
report.Operations = append([]ExecutionOperation(nil), operations...)
report.ExecutedStatements, report.FailedStatements, report.SkippedStatements, report.RolledBackStatements = 0, 0, 0, 0
report.Errors = nil
report.StartTime = time.Now().Format(time.RFC3339Nano)
report.Outcome = "not_started"
for i := range report.Operations {
report.Operations[i].StatementNumber = i + 1
report.Operations[i].Status = "skipped"
}
var runErr error
atomic := opts.AtomicExecution
if atomic && opts.ShouldContinue() {
runErr = errors.New("atomic execution cannot be combined with continue-on-error")
} else if atomic && (backend.Begin == nil || backend.Commit == nil || backend.Rollback == nil) {
runErr = errors.New("atomic execution is not supported by this executor")
} else if atomic {
runErr = backend.Begin(ctx)
}
if runErr == nil {
report.Outcome = "nontransactional"
if backend.ScriptManaged {
report.Outcome = "script_managed"
}
stopped := false
for i := range report.Operations {
op := &report.Operations[i]
if stopped || op.Skip || strings.TrimSpace(op.Statement) == "" {
continue
}
if err := backend.Exec(ctx, i, op.Statement); err != nil {
runErr = errors.Join(runErr, err)
op.Status, op.Error = "failed", err.Error()
report.FailedStatements++
report.Errors = append(report.Errors, ExecutionError{i + 1, op.Statement, err.Error()})
stopped = op.StopOnFailure || !opts.ShouldContinue()
} else {
op.Status = "applied"
if backend.ScriptManaged {
op.Status = "executed"
}
report.ExecutedStatements++
}
}
if atomic {
if report.FailedStatements > 0 {
// Cleanup uses a bounded context independent of caller cancellation.
cleanup, cancel := context.WithTimeout(context.Background(), 5*time.Second)
rollbackErr := backend.Rollback(cleanup)
cancel()
if rollbackErr == nil {
report.Outcome = "rolled_back"
for i := range report.Operations {
if report.Operations[i].Status == "applied" {
report.Operations[i].Status = "rolled_back"
report.RolledBackStatements++
}
}
report.ExecutedStatements = 0
} else {
report.Outcome = "unknown"
runErr = errors.Join(runErr, fmt.Errorf("rollback failed: %w", rollbackErr))
}
} else {
runErr = backend.Commit(ctx)
if runErr == nil {
report.Outcome = "committed"
} else {
report.Outcome = "unknown"
}
}
if report.Outcome == "rolled_back" || report.Outcome == "unknown" {
for i := range report.Schemas {
for j := range report.Schemas[i].Tables {
report.Schemas[i].Tables[j].Created = false
}
}
}
if report.Outcome == "unknown" {
for i := range report.Operations {
if report.Operations[i].Status == "applied" {
report.Operations[i].Status = "unknown"
}
}
report.ExecutedStatements = 0
}
}
}
for _, op := range report.Operations {
if op.Status == "skipped" {
report.SkippedStatements++
}
}
report.EndTime = time.Now().Format(time.RFC3339Nano)
if report.FailedStatements > 0 {
runErr = errors.Join(&ExecutionFailure{report}, runErr)
}
return FinishExecution(opts, report, runErr)
}
// FinishExecution publishes legacy counters and a typed report even on failure.
func FinishExecution(opts *WriterOptions, report *ExecutionReport, runErr error) error {
if opts.Metadata == nil {
opts.Metadata = map[string]interface{}{}
}
opts.Metadata["execution_report"] = report
opts.Metadata["execution_total"] = report.TotalStatements
opts.Metadata["execution_success"] = report.ExecutedStatements
opts.Metadata["execution_failed"] = report.FailedStatements
opts.Metadata["execution_skipped"] = report.SkippedStatements
opts.Metadata["execution_rolled_back"] = report.RolledBackStatements
if path, _ := opts.Metadata["report_path"].(string); path != "" {
if err := WriteExecutionReport(path, report); err != nil {
runErr = errors.Join(runErr, fmt.Errorf("failed to write requested execution report: %w", err))
}
}
return runErr
}
func WriteExecutionReport(path string, report *ExecutionReport) error {
file, err := os.Create(path)
if err != nil {
return fmt.Errorf("failed to create report file: %w", err)
}
encoder := json.NewEncoder(file)
encoder.SetIndent("", " ")
return errors.Join(encoder.Encode(report), file.Close())
}
// StatementOperations creates a plan from the writers' structured statement lists.
func StatementOperations(statements []string) []ExecutionOperation {
ops := make([]ExecutionOperation, len(statements))
for i, stmt := range statements {
ops[i] = ExecutionOperation{Statement: stmt, StopOnFailure: strings.HasPrefix(strings.TrimSpace(stmt), "DO $relspec_preflight"), Skip: strings.TrimSpace(stmt) == "" || strings.HasPrefix(strings.TrimSpace(stmt), "--")}
}
return ops
}
// FailExecution records an invocation that failed before issuing any SQL.
func FailExecution(opts *WriterOptions, dialect string, operations []ExecutionOperation, cause error) error {
report := &ExecutionReport{Dialect: dialect, TotalStatements: len(operations), SkippedStatements: len(operations), Outcome: "not_started", Operations: append([]ExecutionOperation(nil), operations...)}
report.StartTime = time.Now().Format(time.RFC3339Nano)
report.EndTime = report.StartTime
for i := range report.Operations {
report.Operations[i].StatementNumber = i + 1
report.Operations[i].Status = "skipped"
}
return FinishExecution(opts, report, cause)
}
+96
View File
@@ -0,0 +1,96 @@
package writers
import (
"context"
"errors"
"path/filepath"
"strings"
"testing"
)
func TestExecutionOutcomes(t *testing.T) {
for _, atomic := range []bool{false, true} {
for _, continuation := range []bool{false, true} {
t.Run(strings.Join([]string{map[bool]string{false: "independent", true: "atomic"}[atomic], map[bool]string{false: "stop", true: "continue"}[continuation]}, "/"), func(t *testing.T) {
attempted, rollbacks := 0, 0
cause := errors.New("injected failure")
backend := ExecutionBackend{
Exec: func(_ context.Context, i int, _ string) error {
attempted++
if i == 1 {
return cause
}
return nil
},
Begin: func(context.Context) error { return nil }, Commit: func(context.Context) error { t.Fatal("must not commit"); return nil },
Rollback: func(context.Context) error { rollbacks++; return nil },
}
opts := &WriterOptions{ContinueOnError: continuation, AtomicExecution: atomic}
report := &ExecutionReport{Dialect: "test"}
err := RunExecution(context.Background(), opts, report, StatementOperations([]string{"one", "bad", "three"}), backend)
if atomic && continuation {
if err == nil || attempted != 0 || report.SkippedStatements != 3 {
t.Fatalf("invalid options must not execute: %+v %v", report, err)
}
return
}
var failure *ExecutionFailure
if !errors.As(err, &failure) || !errors.Is(err, cause) {
t.Fatalf("must preserve typed report and cause: %v", err)
}
if report.FailedStatements != 1 || report.TotalStatements != 3 || report.Errors[0].StatementNumber != 2 {
t.Fatalf("failure report: %+v", report)
}
if atomic {
if rollbacks != 1 || report.ExecutedStatements != 0 || report.RolledBackStatements != 1 || report.Operations[0].Status != "rolled_back" || report.Outcome != "rolled_back" {
t.Fatalf("rollback report: %+v", report)
}
} else if continuation {
if attempted != 3 || report.ExecutedStatements != 2 || report.SkippedStatements != 0 {
t.Fatalf("continuation report: %+v", report)
}
} else if attempted != 2 || report.ExecutedStatements != 1 || report.SkippedStatements != 1 {
t.Fatalf("stop report: %+v", report)
}
if opts.Metadata["execution_report"] != report {
t.Fatal("typed results missing")
}
})
}
}
}
func TestExecutionUncertainFinalization(t *testing.T) {
for _, failStatement := range []bool{false, true} {
t.Run(map[bool]string{false: "commit", true: "rollback"}[failStatement], func(t *testing.T) {
finalErr := errors.New("connection lost during finalization")
backend := ExecutionBackend{
Begin: func(context.Context) error { return nil },
Exec: func(_ context.Context, i int, _ string) error {
if failStatement && i == 1 {
return errors.New("statement failed")
}
return nil
},
Commit: func(context.Context) error { return finalErr }, Rollback: func(context.Context) error { return finalErr },
}
report := &ExecutionReport{}
err := RunExecution(context.Background(), &WriterOptions{AtomicExecution: true}, report, StatementOperations([]string{"one", "two"}), backend)
if !errors.Is(err, finalErr) || report.Outcome != "unknown" || report.Operations[0].Status != "unknown" || report.RolledBackStatements != 0 || report.ExecutedStatements != 0 {
t.Fatalf("uncertain outcome misreported: %+v %v", report, err)
}
})
}
}
func TestExecutionReportFailureAndFatalPreflight(t *testing.T) {
opts := &WriterOptions{ContinueOnError: true, Metadata: map[string]interface{}{"report_path": filepath.Join(t.TempDir(), "missing", "report.json")}}
report := &ExecutionReport{}
calls := 0
ops := []ExecutionOperation{{Statement: "preflight", StopOnFailure: true}, {Statement: "must not run"}}
err := RunExecution(context.Background(), opts, report, ops, ExecutionBackend{Exec: func(context.Context, int, string) error { calls++; return errors.New("invalid data") }})
var failure *ExecutionFailure
if !errors.As(err, &failure) || !strings.Contains(err.Error(), "requested execution report") || calls != 1 || report.SkippedStatements != 1 {
t.Fatalf("must retain both failures and stop: %+v %v", report, err)
}
}
+15 -15
View File
@@ -502,30 +502,30 @@ func (w *Writer) executeDatabaseSQL(db *models.Database, connString string) erro
// Connect to database // Connect to database
dbConn, err := sql.Open("mssql", connString) dbConn, err := sql.Open("mssql", connString)
if err != nil { if err != nil {
return fmt.Errorf("failed to connect to database: %w", err) return writers.FailExecution(w.options, "SQL Server", writers.StatementOperations(statements), fmt.Errorf("failed to connect to database: %w", err))
} }
defer dbConn.Close() defer dbConn.Close()
ctx := context.Background() ctx := context.Background()
if err = dbConn.PingContext(ctx); err != nil { if err = dbConn.PingContext(ctx); err != nil {
return fmt.Errorf("failed to ping database: %w", err) return writers.FailExecution(w.options, "SQL Server", writers.StatementOperations(statements), fmt.Errorf("failed to ping database: %w", err))
} }
// Execute statements var tx *sql.Tx
executedCount := 0 backend := writers.ExecutionBackend{
for i, stmt := range statements { Exec: func(ctx context.Context, _ int, stmt string) error {
fmt.Fprintf(os.Stderr, "Executing statement %d/%d...\n", i+1, len(statements)) if tx != nil {
_, err := tx.ExecContext(ctx, stmt)
if _, execErr := dbConn.ExecContext(ctx, stmt); execErr != nil { return err
fmt.Fprintf(os.Stderr, "⚠ Warning: Statement failed: %v\n", execErr)
continue
} }
_, err := dbConn.ExecContext(ctx, stmt)
executedCount++ return err
},
Begin: func(ctx context.Context) error { var err error; tx, err = dbConn.BeginTx(ctx, nil); return err },
Commit: func(context.Context) error { return tx.Commit() },
Rollback: func(context.Context) error { return tx.Rollback() },
} }
return writers.RunExecution(ctx, w.options, &writers.ExecutionReport{Dialect: "SQL Server"}, writers.StatementOperations(statements), backend)
fmt.Fprintf(os.Stderr, "✓ Successfully executed %d statements\n", executedCount)
return nil
} }
// generateStatements renders the same script WriteDatabase would produce and // generateStatements renders the same script WriteDatabase would produce and
+69
View File
@@ -0,0 +1,69 @@
package mssql
import (
"database/sql"
"errors"
"fmt"
"os"
"testing"
"time"
"git.warky.dev/wdevs/relspecgo/pkg/models"
"git.warky.dev/wdevs/relspecgo/pkg/writers"
)
func TestLive_ExecutionFailureModes(t *testing.T) {
connection := os.Getenv("RELSPEC_TEST_MSSQL_CONN")
if connection == "" {
t.Skip("RELSPEC_TEST_MSSQL_CONN not set")
}
conn, err := sql.Open("mssql", connection)
if err != nil {
t.Fatal(err)
}
defer conn.Close()
for _, mode := range []string{"stop", "continue", "atomic"} {
t.Run(mode, func(t *testing.T) {
prefix := fmt.Sprintf("audit_%d_", time.Now().UnixNano())
db := models.InitDatabase("audit")
schema := models.InitSchema("dbo")
for _, name := range []string{"first", "bad", "last"} {
table := models.InitTable(prefix+name, "dbo")
col := models.InitColumn("id", table.Name, "dbo")
col.Type = "integer"
if name == "bad" {
duplicate := models.InitColumn("id", table.Name, "dbo")
duplicate.Type = "integer"
table.Columns["duplicate"] = duplicate
}
table.Columns[col.Name] = col
schema.Tables = append(schema.Tables, table)
}
db.Schemas = append(db.Schemas, schema)
t.Cleanup(func() {
for _, name := range []string{"first", "bad", "last"} {
_, _ = conn.Exec("DROP TABLE IF EXISTS dbo." + prefix + name)
}
})
opts := &writers.WriterOptions{ContinueOnError: mode == "continue", AtomicExecution: mode == "atomic", Metadata: map[string]interface{}{"connection_string": connection}}
err := NewWriter(opts).WriteDatabase(db)
var failure *writers.ExecutionFailure
if !errors.As(err, &failure) || failure.Report.FailedStatements == 0 {
t.Fatalf("expected structured error: %v", err)
}
for _, name := range []string{"first", "last"} {
var count int
if err := conn.QueryRow("SELECT count(*) FROM sys.tables WHERE name=?", prefix+name).Scan(&count); err != nil {
t.Fatal(err)
}
want := mode != "atomic" && (name == "first" || mode == "continue")
if (count == 1) != want {
t.Fatalf("%s exists=%v want=%v", name, count == 1, want)
}
}
if mode == "atomic" && (failure.Report.Outcome != "rolled_back" || failure.Report.RolledBackStatements == 0 || failure.Report.ExecutedStatements != 0) {
t.Fatalf("incorrect rollback report: %+v", failure.Report)
}
})
}
}
+145
View File
@@ -0,0 +1,145 @@
package mysql
import (
"os"
"path/filepath"
"strings"
"testing"
"git.warky.dev/wdevs/relspecgo/pkg/models"
"git.warky.dev/wdevs/relspecgo/pkg/transform"
"git.warky.dev/wdevs/relspecgo/pkg/writers"
)
func TestStrictConversionPreflightsAllObjects(t *testing.T) {
db := shopDB()
unsupported := models.InitTable("unsupported", "shop")
col := models.InitColumn("opaque", "unsupported", "shop")
col.Type = "unknown_type"
unsupported.Columns[col.Name] = col
index := models.InitIndex("partial", "unsupported", "shop")
index.Columns = []string{col.Name}
index.Where = "opaque IS NOT NULL"
index.Include = []string{"other"}
unsupported.Indexes[index.Name] = index
db.Schemas[0].Tables = append(db.Schemas[0].Tables, unsupported)
out := filepath.Join(t.TempDir(), "existing.sql")
if err := os.WriteFile(out, []byte("keep existing contents"), 0o600); err != nil {
t.Fatal(err)
}
opts := &writers.WriterOptions{StrictConversion: true, OutputPath: out}
err := NewWriter(opts).WriteDatabase(db)
for _, want := range []string{"lossy MySQL conversion rejected", "unknown type", "partial predicate", "INCLUDE"} {
if err == nil || !strings.Contains(err.Error(), want) {
t.Fatalf("expected %s: %v", want, err)
}
}
bytes, err := os.ReadFile(out)
if err != nil || string(bytes) != "keep existing contents" {
t.Fatalf("destination changed: %q %v", bytes, err)
}
opts.Metadata = map[string]any{"connection_string": "invalid DSN"}
if err := NewWriter(opts).WriteDatabase(db); err == nil || !strings.Contains(err.Error(), "lossy MySQL") {
t.Fatalf("connected before strict validation: %v", err)
}
opts.StrictConversion = false
opts.Metadata = nil
var warnings []string
opts.Warning = func(s string) { warnings = append(warnings, s) }
if err := NewWriter(opts).WriteDatabase(db); err != nil {
t.Fatal(err)
}
if len(warnings) != 3 {
t.Fatalf("warnings: %v", warnings)
}
}
func TestUnsupportedModelFeaturesDiagnosed(t *testing.T) {
db := shopDB()
s := db.Schemas[0]
table := s.Tables[0]
s.Metadata["mysql_unsupported"] = []any{"trigger shop.audit", "partitioned table shop.archive"}
c := table.Columns["id"]
c.Type = "timestamptz"
c.Identity = true
c.Metadata["mysql_srs_id"] = 4326
check := models.InitConstraint("deferred", models.CheckConstraint)
check.Expression = "id > 0"
yes := true
check.Deferrable = &yes
table.Constraints[check.Name] = check
i := models.InitIndex("gin", "users", "shop")
i.Type = "gin"
i.Columns = []string{"id"}
table.Indexes[i.Name] = i
_, err := NewWriter(&writers.WriterOptions{StrictConversion: true}).Statements(db)
for _, want := range []string{"trigger shop.audit", "partitioned table", "time zone", "identity", "mysql_srs_id", "deferrable", "unsupported index type gin"} {
if err == nil || !strings.Contains(err.Error(), want) {
t.Errorf("missing diagnostic %q: %v", want, err)
}
}
}
func TestCanonicalDimensionsAndNativeEdit(t *testing.T) {
db := shopDB()
c := db.Schemas[0].Tables[0].Columns["age"]
c.Type = "decimal"
c.Precision = 12
c.Scale = 4
if got := columnType(c); got != "DECIMAL(12,4)" {
t.Fatal(got)
}
c.Type = "uint64"
c.Precision = 0
c.Scale = 0
if got := columnType(c); got != "BIGINT UNSIGNED" {
t.Fatal(got)
}
c.Metadata["mysql_type"] = "bigint unsigned"
c.Metadata["mysql_type_signature"] = signature(c)
c.Type = "int64"
_, err := NewWriter(&writers.WriterOptions{StrictConversion: true}).Statements(db)
if err == nil || !strings.Contains(err.Error(), "override native declaration") {
t.Fatalf("native metadata silently overrode edit: %v", err)
}
if got := columnType(c); got != "BIGINT" {
t.Fatal(got)
}
}
func TestLowercaseIndexPartsPreserveSQL(t *testing.T) {
db := shopDB()
table := db.Schemas[0].Tables[0]
c := models.InitColumn("Mixed", "users", "shop")
c.Type = "string"
table.Columns[c.Name] = c
i := models.InitIndex("MixedIndex", "users", "shop")
i.KeyParts = []models.IndexPart{{Name: "Mixed", Length: 8, Descending: true}, {Expression: "lower(`Mixed`)"}}
table.Indexes[i.Name] = i
model, err := transform.LowercaseIdentifiers(db)
if err != nil {
t.Fatal(err)
}
parts := model.Schemas[0].Tables[0].Indexes["mixedindex"].KeyParts
if parts[0].Name != "mixed" || parts[1].Expression != "lower(`Mixed`)" || i.KeyParts[0].Name != "Mixed" {
t.Fatalf("lowercase parts: %+v", parts)
}
}
func TestLowercasePrimaryIndexIsNotDuplicated(t *testing.T) {
db := shopDB()
table := db.Schemas[0].Tables[0]
i := models.InitIndex("PRIMARY", table.Name, table.Schema)
i.Unique = true
i.Type = "btree"
i.Columns = []string{"id"}
i.KeyParts = []models.IndexPart{{Name: "id", Descending: true}}
table.Indexes[i.Name] = i
statements, err := NewWriter(&writers.WriterOptions{StrictConversion: true, LowercaseIdentifiers: true}).Statements(db)
if err != nil {
t.Fatal(err)
}
if !strings.Contains(statements[0], "PRIMARY KEY (`id` DESC)") || strings.Contains(statements[0], "KEY `primary`") {
t.Fatalf("primary key duplicated or attributes lost: %s", statements[0])
}
}
+184
View File
@@ -0,0 +1,184 @@
package mysql
import (
"fmt"
"sort"
"strings"
"git.warky.dev/wdevs/relspecgo/pkg/mariadb"
"git.warky.dev/wdevs/relspecgo/pkg/models"
)
// Diagnostics cover modeled schema semantics and features discovered by the
// MySQL reader. They do not certify arbitrary SQL expressions or catalog grants.
func conversionDiagnostics(db *models.Database) []string {
var out []string
warn := func(path, feature string) { out = append(out, path+": "+feature) }
unsupported := func(m map[string]any) {
switch values := m["mysql_unsupported"].(type) {
case []string:
for _, v := range values {
warn(v, "unsupported object")
}
case []any:
for _, v := range values {
warn(fmt.Sprint(v), "unsupported object")
}
}
}
metadata := func(path string, m map[string]any, allowed string) {
if len(models.GetDirectives(m)) > 0 {
warn(path, "dialect directives are not emitted")
}
for _, key := range keys(m) {
if strings.HasPrefix(key, "mysql_") && !strings.Contains(" "+allowed+" ", " "+key+" ") {
warn(path, "unsupported metadata "+key+": "+fmt.Sprint(m[key]))
}
}
}
metadata("database", db.Metadata, "mysql_unsupported mysql_sql_mode")
modeWarning := func(path string, m map[string]any) {
if mode := unsupportedMode(metaString(m, "mysql_sql_mode")); mode != "" {
warn(path, "unsupported source SQL mode "+mode+"; generated SQL requires standard quotes and backslash escaping")
}
}
modeWarning("database", db.Metadata)
unsupported(db.Metadata)
if len(db.Domains) > 0 {
warn("database", "domains are not emitted")
}
for _, s := range db.Schemas {
path := s.Name
metadata(path, s.Metadata, "mysql_unsupported mysql_sql_mode")
modeWarning(path, s.Metadata)
unsupported(s.Metadata)
if len(s.Views) > 0 {
warn(path, "views are not emitted")
}
if len(s.Sequences) > 0 {
warn(path, "standalone sequences are not emitted")
}
if len(s.Enums) > 0 {
warn(path, "standalone enum types are not emitted; use native column enum types")
}
if len(s.Scripts) > 0 {
warn(path, "scripts are not emitted")
}
if s.Owner != "" || len(s.Permissions) > 0 {
warn(path, "ownership and permissions are not emitted")
}
for _, t := range s.Tables {
tp := path + "." + t.Name
modeWarning(tp, t.Metadata)
if t.Tablespace != "" {
warn(tp, "tablespace is not emitted")
}
metadata(tp, t.Metadata, "mysql_engine mysql_collation mysql_create_options mysql_auto_increment mysql_sql_mode")
if options := metaString(t.Metadata, "mysql_create_options"); options != "" {
warn(tp, "table CREATE_OPTIONS are not emitted: "+options)
}
for _, name := range keys(t.Columns) {
c := t.Columns[name]
cp := tp + "." + c.Name
metadata(cp, c.Metadata, "mysql_type mysql_type_signature mysql_charset mysql_datetime_precision mysql_default_expression mysql_default_sql mysql_default_value mysql_generation_storage mysql_invisible mysql_on_update")
if _, known := mariadb.CanonicalToMariaDBTypes[strings.ToLower(c.Type)]; !known && metaString(c.Metadata, "mysql_type") == "" {
warn(cp, "unknown type "+c.Type+" maps to "+columnType(c))
}
if native := metaString(c.Metadata, "mysql_type"); native != "" && metaString(c.Metadata, "mysql_type_signature") != signature(c) {
warn(cp, "edited canonical type/dimensions override native declaration "+native)
}
if c.Type == "timestamptz" {
warn(cp, "time zone semantics are not supported")
}
if c.Type == "jsonb" {
warn(cp, "jsonb storage semantics become JSON")
}
if c.Identity {
warn(cp, "identity semantics are not emitted; use auto_increment")
}
if c.Generated && c.GenerationExpression == "" {
warn(cp, "generated column has no expression")
}
if !c.Generated && c.GenerationExpression != "" {
warn(cp, "generation expression requires generated=true")
}
if c.Generated && (c.Default != nil || c.AutoIncrement) {
warn(cp, "generated column cannot preserve a default or auto_increment")
}
if storage := metaString(c.Metadata, "mysql_generation_storage"); storage != "" && storage != "STORED" && storage != "VIRTUAL" {
warn(cp, "unsupported generation storage "+storage)
}
}
for _, name := range keys(t.Constraints) {
c := t.Constraints[name]
cp := tp + " constraint " + c.Name
metadata(cp, c.Metadata, "mysql_enforced")
if c.Deferrable != nil && *c.Deferrable || c.InitiallyDeferred != nil && *c.InitiallyDeferred {
warn(cp, "deferrable constraints are not supported")
}
switch c.Type {
case models.PrimaryKeyConstraint, models.UniqueConstraint:
case models.CheckConstraint:
if c.Expression == "" {
warn(cp, "check constraint has no expression")
}
case models.ForeignKeyConstraint:
engine := metaString(t.Metadata, "mysql_engine")
if engine != "" && !strings.EqualFold(engine, "InnoDB") && !strings.EqualFold(engine, "NDB") {
warn(cp, "foreign keys are not supported by engine "+engine)
}
for _, action := range []string{c.OnDelete, c.OnUpdate} {
switch strings.ToUpper(action) {
case "", "CASCADE", "SET NULL", "RESTRICT", "NO ACTION":
default:
warn(cp, "unsupported referential action is not emitted: "+action)
}
}
default:
warn(cp, "unsupported constraint type "+string(c.Type))
}
}
for _, name := range keys(t.Indexes) {
i := t.Indexes[name]
ip := tp + " index " + i.Name
metadata(ip, i.Metadata, "mysql_visible")
if i.Where != "" {
warn(ip, "partial predicate is not emitted")
}
if len(i.Include) > 0 {
warn(ip, "INCLUDE columns are not emitted")
}
if i.Concurrent {
warn(ip, "concurrent creation is not supported")
}
if strings.EqualFold(i.Type, "hash") && !strings.EqualFold(metaString(t.Metadata, "mysql_engine"), "MEMORY") {
warn(ip, "HASH index requires a compatible engine; InnoDB converts it to BTREE")
}
switch strings.ToUpper(i.Type) {
case "", "BTREE", "HASH", "FULLTEXT", "SPATIAL":
default:
warn(ip, "unsupported index type "+i.Type+"; index is not emitted")
}
for _, p := range i.KeyParts {
if (p.Name == "") == (p.Expression == "") {
warn(ip, "key part requires exactly one name or expression")
}
}
}
for _, name := range keys(t.Relationships) {
r := t.Relationships[name]
if t.Constraints[r.ForeignKey] == nil {
warn(tp+" relationship "+r.Name, "no backing foreign key constraint; relationship is not emitted")
}
}
}
}
sort.Strings(out)
unique := out[:0]
for _, d := range out {
if len(unique) == 0 || unique[len(unique)-1] != d {
unique = append(unique, d)
}
}
return unique
}
+142
View File
@@ -0,0 +1,142 @@
package mysql_test
import (
"database/sql"
"fmt"
"os"
"path/filepath"
"strings"
"testing"
"time"
driver "github.com/go-sql-driver/mysql"
"git.warky.dev/wdevs/relspecgo/pkg/models"
"git.warky.dev/wdevs/relspecgo/pkg/readers"
rmysql "git.warky.dev/wdevs/relspecgo/pkg/readers/mysql"
"git.warky.dev/wdevs/relspecgo/pkg/writers"
wmysql "git.warky.dev/wdevs/relspecgo/pkg/writers/mysql"
)
func TestLiveUnsupportedObjectsDiagnosed(t *testing.T) {
conn := os.Getenv("RELSPEC_TEST_MYSQL_CONN")
if conn == "" {
t.Skip("RELSPEC_TEST_MYSQL_CONN is not set")
}
db, err := sql.Open("mysql", conn)
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { db.Close() })
prefix := fmt.Sprintf("unsupported_%d", time.Now().UnixNano())
table, view, trigger := prefix+"_table", prefix+"_view", prefix+"_trigger"
t.Cleanup(func() {
for _, stmt := range []string{"DROP VIEW IF EXISTS " + view, "DROP TRIGGER IF EXISTS " + trigger, "DROP TABLE IF EXISTS " + table} {
if _, err := db.Exec(stmt); err != nil {
t.Error(err)
}
}
})
for _, stmt := range []string{
"CREATE TABLE " + table + " (id INT PRIMARY KEY) PARTITION BY HASH(id) PARTITIONS 2",
"CREATE VIEW " + view + " AS SELECT id FROM " + table,
"CREATE TRIGGER " + trigger + " BEFORE INSERT ON " + table + " FOR EACH ROW SET NEW.id = NEW.id + 1",
} {
if _, err := db.Exec(stmt); err != nil {
t.Fatalf("%s: %v", stmt, err)
}
}
model, err := rmysql.NewReader(&readers.ReaderOptions{ConnectionString: conn}).ReadDatabase()
if err != nil {
t.Fatal(err)
}
path := filepath.Join(t.TempDir(), "schema.sql")
if err := os.WriteFile(path, []byte("original"), 0o600); err != nil {
t.Fatal(err)
}
err = wmysql.NewWriter(&writers.WriterOptions{StrictConversion: true, OutputPath: path}).WriteDatabase(model)
for _, want := range []string{"partitioned table", view, trigger} {
if err == nil || !strings.Contains(err.Error(), want) {
t.Errorf("missing diagnostic %s: %v", want, err)
}
}
bytes, readErr := os.ReadFile(path)
if readErr != nil || string(bytes) != "original" {
t.Fatalf("strict failure touched file: %s %v", bytes, readErr)
}
// Schema-filtered/library writes retain discovery diagnostics as well.
err = wmysql.NewWriter(&writers.WriterOptions{StrictConversion: true}).WriteSchema(model.Schemas[0])
if err == nil || !strings.Contains(err.Error(), trigger) {
t.Fatalf("filtered schema lost diagnostics: %v", err)
}
var warnings []string
_, err = wmysql.NewWriter(&writers.WriterOptions{Warning: func(s string) { warnings = append(warnings, s) }}).Statements(model)
if err != nil {
t.Fatal(err)
}
for _, want := range []string{view, trigger, table} {
found := false
for _, warning := range warnings {
if strings.Contains(warning, want) {
found = true
}
}
if !found {
t.Errorf("no warning for %s: %v", want, warnings)
}
}
}
func TestLiveUnsupportedSQLModeFailsBeforeDDL(t *testing.T) {
conn := os.Getenv("RELSPEC_TEST_MYSQL_CONN")
if conn == "" {
t.Skip("RELSPEC_TEST_MYSQL_CONN is not set")
}
for _, mode := range []string{"NO_BACKSLASH_ESCAPES", "ANSI_QUOTES"} {
t.Run(mode, func(t *testing.T) {
cfg, err := driver.ParseDSN(conn)
if err != nil {
t.Fatal(err)
}
if cfg.Params == nil {
cfg.Params = map[string]string{}
}
cfg.Params["sql_mode"] = "'" + mode + "'"
modeConn := cfg.FormatDSN()
model, err := rmysql.NewReader(&readers.ReaderOptions{ConnectionString: modeConn}).ReadDatabase()
if err != nil {
t.Fatal(err)
}
if _, err := wmysql.NewWriter(&writers.WriterOptions{StrictConversion: true}).Statements(model); err == nil || !strings.Contains(err.Error(), mode) {
t.Fatalf("source mode not diagnosed: %v", err)
}
db, err := sql.Open("mysql", conn)
if err != nil {
t.Fatal(err)
}
defer db.Close()
name := fmt.Sprintf("mode_%d", time.Now().UnixNano())
defer db.Exec("DROP TABLE IF EXISTS " + name)
target := models.InitDatabase(cfg.DBName)
s := models.InitSchema(cfg.DBName)
target.Schemas = append(target.Schemas, s)
table := models.InitTable(name, s.Name)
s.Tables = append(s.Tables, table)
c := models.InitColumn("id", name, s.Name)
c.Type = "int"
table.Columns[c.Name] = c
err = wmysql.NewWriter(&writers.WriterOptions{StrictConversion: true, Metadata: map[string]any{"connection_string": modeConn}}).WriteDatabase(target)
if err == nil || !strings.Contains(err.Error(), mode) {
t.Fatalf("target mode not rejected: %v", err)
}
var count int
if err := db.QueryRow("SELECT COUNT(*) FROM information_schema.TABLES WHERE TABLE_SCHEMA=? AND TABLE_NAME=?", cfg.DBName, name).Scan(&count); err != nil {
t.Fatal(err)
}
if count != 0 {
t.Fatal("mode rejection occurred after DDL")
}
})
}
}
+244
View File
@@ -0,0 +1,244 @@
package mysql_test
import (
"database/sql"
"encoding/json"
"fmt"
"os"
"reflect"
"sort"
"strings"
"testing"
"time"
"gopkg.in/yaml.v3"
driver "github.com/go-sql-driver/mysql"
"git.warky.dev/wdevs/relspecgo/pkg/models"
"git.warky.dev/wdevs/relspecgo/pkg/readers"
rmysql "git.warky.dev/wdevs/relspecgo/pkg/readers/mysql"
"git.warky.dev/wdevs/relspecgo/pkg/writers"
wmysql "git.warky.dev/wdevs/relspecgo/pkg/writers/mysql"
)
func TestLiveSchemaRoundTrip(t *testing.T) {
conn := os.Getenv("RELSPEC_TEST_MYSQL_CONN")
if conn == "" {
t.Skip("RELSPEC_TEST_MYSQL_CONN is not set")
}
db, err := sql.Open("mysql", conn)
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { db.Close() })
prefix := fmt.Sprintf("rt_%d", time.Now().UnixNano())
parent, child := prefix+"_zparent", prefix+"_achild"
q := func(n string) string { return "`" + strings.ReplaceAll(n, "`", "``") + "`" }
exec := func(stmt string) {
t.Helper()
if _, err := db.Exec(stmt); err != nil {
t.Fatalf("%v\n%s", err, stmt)
}
}
t.Cleanup(func() {
for _, name := range []string{child + "_copy", child, parent + "_copy", parent} {
if _, err := db.Exec("DROP TABLE IF EXISTS " + q(name)); err != nil {
t.Errorf("cleanup %s: %v", name, err)
}
}
})
exec("CREATE TABLE " + q(parent) + ` (
id BIGINT UNSIGNED NOT NULL AUTO_INCREMENT,
code CHAR(3) NOT NULL,
email VARCHAR(100) NOT NULL,
label VARCHAR(64) DEFAULT 'don''t (change)' COMMENT 'column''s comment',
amount DECIMAL(12,4) NOT NULL DEFAULT 3.2500,
medium MEDIUMINT UNSIGNED,
flags BIT(8) DEFAULT b'00001010',
status ENUM('one','two','don''t, split') DEFAULT 'don''t, split',
choices SET('a','b') DEFAULT 'a,b',
stamp TIMESTAMP(6) NULL DEFAULT CURRENT_TIMESTAMP(6) ON UPDATE CURRENT_TIMESTAMP(6),
duration TIME(4),
bytes VARBINARY(16),
binary_def VARBINARY(8) DEFAULT X'00275CFF',
slash VARCHAR(64) DEFAULT 'C:\\temp\\file',
uuid_value CHAR(36) DEFAULT (uuid()),
hidden_col INT INVISIBLE,
dt DATETIME(6) DEFAULT '2020-01-02 03:04:05.123456',
payload MEDIUMBLOB,
doc TEXT DEFAULT ('first;
-- data, not a comment
last'),
generated_amount DECIMAL(13,4) GENERATED ALWAYS AS (amount * 2) STORED,
virtual_label VARCHAR(80) GENERATED ALWAYS AS (concat('prefix:',lower(label))) VIRTUAL,
PRIMARY KEY (id,code),
UNIQUE KEY email_unique (email),
UNIQUE KEY pair_unique (label,email),
UNIQUE KEY unique_expression ((concat(lower(code),'-x')),email),
KEY prefix_desc (label(10) DESC),
KEY expression_key ((lower(email))),
KEY invisible_key (medium) INVISIBLE COMMENT 'index''s comment',
FULLTEXT KEY text_search (doc),
CONSTRAINT ` + q(prefix+"_amount_check") + ` CHECK (amount >= 0),
CONSTRAINT ` + q(prefix+"_optional_check") + ` CHECK (medium > 0 AND status <> 'forbidden;-- value') NOT ENFORCED
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci COMMENT='table''s comment'`)
exec("CREATE TABLE " + q(child) + ` (
id INT PRIMARY KEY,
parent_id BIGINT UNSIGNED,
parent_code CHAR(3),
parent_email VARCHAR(100),
CONSTRAINT ` + q(prefix+"_composite_fk") + ` FOREIGN KEY (parent_id,parent_code) REFERENCES ` + q(parent) + ` (id,code) ON DELETE SET NULL ON UPDATE CASCADE,
CONSTRAINT ` + q(prefix+"_email_fk") + ` FOREIGN KEY (parent_email) REFERENCES ` + q(parent) + ` (email) ON DELETE CASCADE ON UPDATE RESTRICT
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci`)
readerConfig, err := driver.ParseDSN(conn)
if err != nil {
t.Fatal(err)
}
if readerConfig.Params == nil {
readerConfig.Params = map[string]string{}
}
readerConfig.Params["sql_quote_show_create"] = "0" // reader must ensure quoted native metadata
read := func() *models.Database {
t.Helper()
model, err := rmysql.NewReader(&readers.ReaderOptions{ConnectionString: readerConfig.FormatDSN()}).ReadDatabase()
if err != nil {
t.Fatal(err)
}
return model
}
model := read()
var selected []*models.Table
for _, table := range model.Schemas[0].Tables {
if table.Name == parent || table.Name == child {
selected = append(selected, table)
}
}
if len(selected) != 2 {
t.Fatalf("read %d fixture tables", len(selected))
}
model.Schemas[0].Tables = selected
model.Metadata = nil
model.Schemas[0].Metadata = nil // unrelated pre-existing test objects are outside this fixture
for _, table := range selected {
if table.Name == parent {
if table.Columns["id"].Type != "uint64" {
t.Fatal("unsigned type lost")
}
if table.Columns["amount"].Precision != 12 || table.Columns["amount"].Scale != 4 {
t.Fatal("decimal dimensions lost")
}
if table.Indexes["expression_key"].KeyParts[0].Expression == "" {
t.Fatal("functional index expression lost")
}
} else if len(table.Relationships) != 2 {
t.Fatalf("foreign key relationships collapsed: %d", len(table.Relationships))
}
}
// Exercise the serialized model boundary as well as the native reader objects.
encoded, err := json.Marshal(model)
if err != nil {
t.Fatal(err)
}
var target models.Database
if err := json.Unmarshal(encoded, &target); err != nil {
t.Fatal(err)
}
yamlModel, err := yaml.Marshal(&target)
if err != nil {
t.Fatal(err)
}
target = models.Database{}
if err := yaml.Unmarshal(yamlModel, &target); err != nil {
t.Fatal(err)
}
for _, table := range target.Schemas[0].Tables {
table.Name += "_copy"
for _, col := range table.Columns {
col.Table = table.Name
}
for _, index := range table.Indexes {
index.Table = table.Name
}
constraints := map[string]*models.Constraint{}
for _, c := range table.Constraints {
c.Table = table.Name
if c.Type == models.ForeignKeyConstraint {
c.ReferencedTable += "_copy"
}
if c.Type == models.ForeignKeyConstraint || c.Type == models.CheckConstraint {
c.Name += "_copy"
}
constraints[c.Name] = c
}
table.Constraints = constraints
for _, r := range table.Relationships {
r.FromTable += "_copy"
r.ToTable += "_copy"
r.ForeignKey += "_copy"
}
}
opts := &writers.WriterOptions{StrictConversion: true, Metadata: map[string]any{"connection_string": conn}}
if err := wmysql.NewWriter(opts).WriteDatabase(&target); err != nil {
t.Fatal(err)
}
show := func(table string) string {
t.Helper()
var name, ddl string
if err := db.QueryRow("SHOW CREATE TABLE "+q(table)).Scan(&name, &ddl); err != nil {
t.Fatal(err)
}
ddl = strings.ReplaceAll(ddl, "_copy", "")
// Index creation order is not semantic. Keep column and constraint order intact.
lines := strings.Split(ddl, "\n")
var indexes []string
for _, line := range lines {
if strings.HasPrefix(strings.TrimSpace(line), "KEY ") {
indexes = append(indexes, line)
}
}
sort.Strings(indexes)
n := 0
for i, line := range lines {
if strings.HasPrefix(strings.TrimSpace(line), "KEY ") {
lines[i] = indexes[n]
n++
}
}
return strings.Join(lines, "\n")
}
for _, name := range []string{parent, child} {
a, b := show(name), show(name+"_copy")
if a != b {
t.Errorf("schema mismatch\nsource:\n%s\ntarget:\n%s", a, b)
}
}
again := read()
for _, table := range again.Schemas[0].Tables {
if table.Name == parent+"_copy" && !reflect.DeepEqual(table.Columns["amount"].Metadata, selected[1].Columns["amount"].Metadata) {
t.Error("type metadata changed on second read")
}
}
// Verify constraints and defaults as behavior, after comparing untouched schemas.
exec("INSERT INTO " + q(parent+"_copy") + " (id,code,email) VALUES (18446744073709551614,'abc','one@example.com')")
exec("INSERT INTO " + q(child+"_copy") + " (id,parent_id,parent_code,parent_email) VALUES (1,18446744073709551614,'abc','one@example.com')")
if _, err := db.Exec("INSERT INTO " + q(child+"_copy") + " (id,parent_id,parent_code) VALUES (2,123,'abc')"); err == nil {
t.Error("foreign key is not enforced")
}
if _, err := db.Exec("UPDATE " + q(parent+"_copy") + " SET amount=-1"); err == nil {
t.Error("check is not enforced")
}
var label, doc, binaryHex, slash string
var flags int
if err := db.QueryRow("SELECT label,doc,flags+0,HEX(binary_def),slash FROM "+q(parent+"_copy")).Scan(&label, &doc, &flags, &binaryHex, &slash); err != nil {
t.Fatal(err)
}
for _, table := range read().Schemas[0].Tables {
if table.Name == parent+"_copy" && table.Metadata["mysql_auto_increment"] != "18446744073709551615" {
t.Errorf("large auto-increment counter lost: %v", table.Metadata["mysql_auto_increment"])
}
}
if label != "don't (change)" || doc != "first;\n-- data, not a comment\nlast" || flags != 10 || binaryHex != "00275CFF" || slash != `C:\temp\file` {
t.Errorf("defaults corrupted: %q %q %d %s %q", label, doc, flags, binaryHex, slash)
}
}
+352 -107
View File
@@ -7,12 +7,14 @@ import (
"io" "io"
"os" "os"
"sort" "sort"
"strconv"
"strings" "strings"
_ "github.com/go-sql-driver/mysql" _ "github.com/go-sql-driver/mysql"
"git.warky.dev/wdevs/relspecgo/pkg/mariadb" "git.warky.dev/wdevs/relspecgo/pkg/mariadb"
"git.warky.dev/wdevs/relspecgo/pkg/models" "git.warky.dev/wdevs/relspecgo/pkg/models"
"git.warky.dev/wdevs/relspecgo/pkg/transform"
"git.warky.dev/wdevs/relspecgo/pkg/writers" "git.warky.dev/wdevs/relspecgo/pkg/writers"
) )
@@ -22,83 +24,202 @@ type Writer struct {
} }
func NewWriter(options *writers.WriterOptions) *Writer { return &Writer{options: options} } func NewWriter(options *writers.WriterOptions) *Writer { return &Writer{options: options} }
func (w *Writer) WriteDatabase(db *models.Database) error {
// Statements preflights the whole model, emits conversion diagnostics, and returns
// executable units. No output files or database connections are opened here.
func (w *Writer) Statements(db *models.Database) ([]string, error) {
if w.options == nil { if w.options == nil {
return fmt.Errorf("writer options are required") return nil, fmt.Errorf("writer options are required")
} }
if conn, ok := w.options.Metadata["connection_string"].(string); ok && conn != "" { if db == nil {
return w.execute(db, conn) return nil, fmt.Errorf("database is required")
} }
release, err := w.openOutput() if w.options.LowercaseIdentifiers {
if err != nil { var err error
return err db, err = transform.LowercaseIdentifiers(db)
}
defer release()
return w.writeDatabaseDDL(db)
}
func (w *Writer) writeDatabaseDDL(db *models.Database) error {
if _, err := fmt.Fprintf(w.writer, "-- MySQL Database Schema\n-- Database: %s\n-- Generated by RelSpec\n\n", db.Name); err != nil {
return err
}
for _, s := range db.Schemas {
if err := w.WriteSchema(s); err != nil {
return err
}
}
return nil
}
// openOutput points w.writer at the configured destination (output file or
// stdout) when none is set, and returns a func that releases it again.
func (w *Writer) openOutput() (func(), error) {
if w.writer != nil {
return func() {}, nil
}
if w.options != nil && w.options.OutputPath != "" {
f, err := os.Create(w.options.OutputPath)
if err != nil { if err != nil {
return nil, err return nil, err
} }
w.writer = f
return func() {
f.Close()
w.writer = nil
}, nil
} }
w.writer = os.Stdout diagnostics := conversionDiagnostics(db)
return func() { w.writer = nil }, nil if len(diagnostics) > 0 && w.options.StrictConversion {
return nil, fmt.Errorf("lossy MySQL conversion rejected:\n- %s", strings.Join(diagnostics, "\n- "))
}
for _, d := range diagnostics {
if w.options.Warning != nil {
w.options.Warning(d)
} else {
fmt.Fprintln(os.Stderr, "warning: MySQL conversion: "+d)
}
}
return databaseStatements(db), nil
} }
func (w *Writer) WriteSchema(s *models.Schema) error { func (w *Writer) WriteDatabase(db *models.Database) error {
release, err := w.openOutput() statements, err := w.Statements(db)
if err != nil { if err != nil {
return err return err
} }
defer release() if conn, ok := w.options.Metadata["connection_string"].(string); ok && conn != "" {
for _, t := range s.Tables { return w.execute(statements, conn)
if err := w.writeTable(s, t); err != nil { }
if w.writer != nil {
return writeStatements(w.writer, statements, db.Name)
}
if w.options.OutputPath == "" {
return writeStatements(os.Stdout, statements, db.Name)
}
f, err := os.Create(w.options.OutputPath)
if err != nil {
return err
}
err = writeStatements(f, statements, db.Name)
closeErr := f.Close()
if err != nil {
return err
}
return closeErr
}
func writeStatements(out io.Writer, statements []string, name string) error {
name = strings.NewReplacer("\n", " ", "\r", " ").Replace(name)
if _, err := fmt.Fprintf(out, "-- Database: %s\n", name); err != nil {
return err
}
if _, err := io.WriteString(out, "-- MySQL table schema; generated by RelSpec\n\n"); err != nil {
return err
}
for _, stmt := range statements {
if _, err := io.WriteString(out, stmt); err != nil {
return err return err
} }
} }
return nil return nil
} }
func (w *Writer) WriteTable(t *models.Table) error { func (w *Writer) WriteSchema(s *models.Schema) error {
release, err := w.openOutput() return w.WriteDatabase(&models.Database{Schemas: []*models.Schema{s}})
if err != nil {
return err
}
defer release()
return w.writeTable(nil, t)
} }
func (w *Writer) writeTable(s *models.Schema, t *models.Table) error { func (w *Writer) WriteTable(t *models.Table) error {
name := t.Name return w.WriteSchema(&models.Schema{Name: "", Tables: []*models.Table{t}})
}
func qualified(schema, name string) string {
if schema == "" {
return quote(name)
}
return quote(schema) + "." + quote(name)
}
func keys[T any](m map[string]T) []string {
out := make([]string, 0, len(m))
for k := range m {
out = append(out, k)
}
sort.Strings(out)
return out
}
func metaString(m map[string]any, key string) string { v, _ := m[key].(string); return v }
func metaBool(m map[string]any, key string, def bool) bool {
v, ok := m[key].(bool)
if ok {
return v
}
return def
}
func signature(c *models.Column) string {
return fmt.Sprintf("%s/%d/%d/%d", c.Type, c.Length, c.Precision, c.Scale)
}
func columnType(c *models.Column) string {
if native := metaString(c.Metadata, "mysql_type"); native != "" && metaString(c.Metadata, "mysql_type_signature") == signature(c) {
return native
}
typ := mariadb.ConvertCanonicalToMariaDB(c.Type)
switch strings.ToLower(c.Type) {
case "string":
if c.Length > 0 {
typ = fmt.Sprintf("VARCHAR(%d)", c.Length)
}
case "decimal":
if c.Precision > 0 {
typ = fmt.Sprintf("DECIMAL(%d,%d)", c.Precision, c.Scale)
}
case "timestamp", "time":
if c.Precision > 0 {
typ += fmt.Sprintf("(%d)", c.Precision)
}
}
return typ
}
// MySQL string literals use doubled quotes and escaped backslashes. The output
// assumes the server's standard SQL mode (NO_BACKSLASH_ESCAPES is not enabled).
func literal(s string) string {
return "'" + strings.ReplaceAll(strings.ReplaceAll(s, "\\", "\\\\"), "'", "''") + "'"
}
func defaultSQL(c *models.Column) string {
value := fmt.Sprint(c.Default)
if native := metaString(c.Metadata, "mysql_default_sql"); native != "" && metaString(c.Metadata, "mysql_default_value") == value {
return native
}
if strings.HasPrefix(metaString(c.Metadata, "mysql_type"), "bit(") && strings.HasPrefix(value, "b'") && strings.HasSuffix(value, "'") && strings.Trim(value[2:len(value)-1], "01") == "" {
return value
}
if metaBool(c.Metadata, "mysql_default_expression", false) {
if strings.HasPrefix(strings.ToUpper(value), "CURRENT_TIMESTAMP") {
return value
}
return "(" + value + ")"
}
switch strings.ToLower(c.Type) {
case "text", "json", "jsonb", "bytea":
return "(" + literal(value) + ")"
case "string", "uuid", "date", "time", "timestamp", "timestamptz":
return literal(value)
case "bool", "boolean":
if value == "true" {
return "1"
}
if value == "false" {
return "0"
}
}
// Numeric defaults are values, never executable fragments.
if _, err := strconv.ParseFloat(value, 64); err == nil {
return value
}
return literal(value)
}
func indexParts(i *models.Index) string {
if len(i.KeyParts) == 0 {
return quoted(i.Columns)
}
parts := make([]string, 0, len(i.KeyParts))
for _, p := range i.KeyParts {
s := quote(p.Name)
if p.Expression != "" {
s = "(" + p.Expression + ")"
}
if p.Length > 0 {
s += fmt.Sprintf("(%d)", p.Length)
}
if p.Descending {
s += " DESC"
}
parts = append(parts, s)
}
return strings.Join(parts, ", ")
}
func tableStatement(s *models.Schema, t *models.Table) string {
schema := ""
if s != nil { if s != nil {
name = fmt.Sprintf("%s.%s", quote(s.Name), quote(t.Name)) schema = s.Name
} else {
name = quote(name)
} }
cols := make([]*models.Column, 0, len(t.Columns)) cols := make([]*models.Column, 0, len(t.Columns))
for _, c := range t.Columns { for _, c := range t.Columns {
@@ -113,92 +234,198 @@ func (w *Writer) writeTable(s *models.Schema, t *models.Table) error {
defs := []string{} defs := []string{}
pk := []string{} pk := []string{}
for _, c := range cols { for _, c := range cols {
d := fmt.Sprintf(" %s %s", quote(c.Name), mariadb.ConvertCanonicalToMariaDB(c.Type)) d := " " + quote(c.Name) + " " + columnType(c)
if c.Length > 0 && strings.EqualFold(c.Type, "string") { if charset := metaString(c.Metadata, "mysql_charset"); charset != "" && c.Collation != metaString(t.Metadata, "mysql_collation") {
d = fmt.Sprintf(" %s VARCHAR(%d)", quote(c.Name), c.Length) d += " CHARACTER SET " + quote(charset)
}
if c.Collation != "" && c.Collation != metaString(t.Metadata, "mysql_collation") {
d += " COLLATE " + quote(c.Collation)
}
if c.Generated {
d += " GENERATED ALWAYS AS (" + c.GenerationExpression + ")"
if strings.EqualFold(metaString(c.Metadata, "mysql_generation_storage"), "STORED") {
d += " STORED"
} else {
d += " VIRTUAL"
}
} }
if c.NotNull { if c.NotNull {
d += " NOT NULL" d += " NOT NULL"
} else if !c.Generated {
d += " NULL"
} }
if c.AutoIncrement { if c.AutoIncrement {
d += " AUTO_INCREMENT" d += " AUTO_INCREMENT"
} }
if c.Default != nil { if c.Default != nil && !c.Generated {
d += fmt.Sprintf(" DEFAULT %s", writers.QuoteDefaultValue(fmt.Sprint(c.Default), c.Type)) d += " DEFAULT " + defaultSQL(c)
}
if upd := metaString(c.Metadata, "mysql_on_update"); upd != "" {
d += " ON UPDATE " + upd
}
if metaBool(c.Metadata, "mysql_invisible", false) {
d += " INVISIBLE"
}
if c.Comment != "" {
d += " COMMENT " + literal(c.Comment)
} }
defs = append(defs, d) defs = append(defs, d)
if c.IsPrimaryKey { if c.IsPrimaryKey {
pk = append(pk, quote(c.Name)) pk = append(pk, c.Name)
} }
} }
constraintNames := make([]string, 0, len(t.Constraints)) for _, name := range keys(t.Constraints) {
for name := range t.Constraints {
constraintNames = append(constraintNames, name)
}
sort.Strings(constraintNames)
for _, name := range constraintNames {
c := t.Constraints[name] c := t.Constraints[name]
if c.Type == models.PrimaryKeyConstraint { if c.Type == models.PrimaryKeyConstraint {
pk = nil pk = c.Columns
for _, n := range c.Columns {
pk = append(pk, quote(n))
}
break break
} }
} }
if len(pk) > 0 { var primaryIndex *models.Index
defs = append(defs, " PRIMARY KEY ("+strings.Join(pk, ", ")+")") for name, i := range t.Indexes {
if strings.EqualFold(name, "PRIMARY") {
primaryIndex = i
break
} }
for _, name := range constraintNames { }
if i := primaryIndex; i != nil {
defs = append(defs, " PRIMARY KEY ("+indexParts(i)+")")
} else if len(pk) > 0 {
defs = append(defs, " PRIMARY KEY ("+quoted(pk)+")")
}
for _, name := range keys(t.Constraints) {
c := t.Constraints[name] c := t.Constraints[name]
if c.Type == models.UniqueConstraint { switch c.Type {
defs = append(defs, fmt.Sprintf(" CONSTRAINT %s UNIQUE (%s)", quote(c.Name), quoted(c.Columns))) case models.UniqueConstraint:
if t.Indexes[c.Name] == nil {
defs = append(defs, " CONSTRAINT "+quote(c.Name)+" UNIQUE ("+quoted(c.Columns)+")")
}
case models.CheckConstraint:
d := " CONSTRAINT " + quote(c.Name) + " CHECK (" + c.Expression + ")"
if !metaBool(c.Metadata, "mysql_enforced", true) {
d += " NOT ENFORCED"
}
defs = append(defs, d)
} }
} }
sql := fmt.Sprintf("CREATE TABLE IF NOT EXISTS %s (\n%s\n) ENGINE=InnoDB;\n\n", name, strings.Join(defs, ",\n")) for _, name := range keys(t.Indexes) {
if _, err := io.WriteString(w.writer, sql); err != nil { i := t.Indexes[name]
return err if strings.EqualFold(name, "PRIMARY") {
continue
} }
return nil method := strings.ToUpper(i.Type)
if method != "" && method != "BTREE" && method != "HASH" && method != "FULLTEXT" && method != "SPATIAL" {
continue
}
kind := "KEY"
if i.Unique {
kind = "UNIQUE KEY"
}
if method == "FULLTEXT" || method == "SPATIAL" {
kind = method + " KEY"
}
d := " " + kind + " " + quote(i.Name) + " (" + indexParts(i) + ")"
if method == "HASH" {
d += " USING " + method
}
if !metaBool(i.Metadata, "mysql_visible", true) {
d += " INVISIBLE"
}
if i.Comment != "" {
d += " COMMENT " + literal(i.Comment)
}
defs = append(defs, d)
}
engine := metaString(t.Metadata, "mysql_engine")
if engine == "" {
engine = "InnoDB"
}
tail := " ENGINE=" + quote(engine)
if engine == "InnoDB" {
tail = " ENGINE=InnoDB"
}
if coll := metaString(t.Metadata, "mysql_collation"); coll != "" {
tail += " COLLATE=" + quote(coll)
}
if auto := metaString(t.Metadata, "mysql_auto_increment"); auto != "" {
if _, err := strconv.ParseUint(auto, 10, 64); err == nil {
tail += " AUTO_INCREMENT=" + auto
}
}
if t.Comment != "" {
tail += " COMMENT=" + literal(t.Comment)
}
return "CREATE TABLE IF NOT EXISTS " + qualified(schema, t.Name) + " (\n" + strings.Join(defs, ",\n") + "\n)" + tail + ";\n\n"
} }
func (w *Writer) execute(dbm *models.Database, conn string) error { func databaseStatements(db *models.Database) []string {
var b strings.Builder var statements []string
old := w.writer for _, s := range db.Schemas {
w.writer = &b for _, t := range s.Tables {
if err := w.writeDatabaseDDL(dbm); err != nil { statements = append(statements, tableStatement(s, t))
w.writer = old }
return err }
// References are added after all tables, supporting cycles and forward references.
for _, s := range db.Schemas {
for _, t := range s.Tables {
for _, name := range keys(t.Constraints) {
c := t.Constraints[name]
if c.Type != models.ForeignKeyConstraint {
continue
}
rs := c.ReferencedSchema
if rs == "" {
rs = s.Name
}
stmt := "ALTER TABLE " + qualified(s.Name, t.Name) + " ADD CONSTRAINT " + quote(c.Name) + " FOREIGN KEY (" + quoted(c.Columns) + ") REFERENCES " + qualified(rs, c.ReferencedTable) + " (" + quoted(c.ReferencedColumns) + ")"
if validAction(c.OnDelete) && c.OnDelete != "" {
stmt += " ON DELETE " + strings.ToUpper(c.OnDelete)
}
if validAction(c.OnUpdate) && c.OnUpdate != "" {
stmt += " ON UPDATE " + strings.ToUpper(c.OnUpdate)
}
statements = append(statements, stmt+";\n\n")
}
}
}
return statements
}
func (w *Writer) execute(statements []string, conn string) error {
if w.options.AtomicExecution {
return fmt.Errorf("MySQL DDL implicitly commits; atomic execution is unsupported")
} }
w.writer = old
db, err := sql.Open("mysql", conn) db, err := sql.Open("mysql", conn)
if err != nil { if err != nil {
return fmt.Errorf("failed to connect: %w", err) return fmt.Errorf("failed to connect: %w", err)
} }
defer db.Close() defer db.Close()
for _, stmt := range strings.Split(b.String(), ";\n") { session, err := db.Conn(context.Background())
stmt = stripComments(strings.TrimSpace(stmt)) if err != nil {
if stmt == "" {
continue
}
if _, err := db.ExecContext(context.Background(), stmt); err != nil {
return fmt.Errorf("failed to execute SQL: %w", err) return fmt.Errorf("failed to execute SQL: %w", err)
} }
defer session.Close()
var mode string
if err := session.QueryRowContext(context.Background(), "SELECT @@SESSION.sql_mode").Scan(&mode); err != nil {
return fmt.Errorf("read MySQL SQL mode: %w", err)
}
if mode = unsupportedMode(mode); mode != "" {
return fmt.Errorf("MySQL execution requires standard quotes and backslash escaping; %s is enabled", mode)
}
var failures []string
for n, stmt := range statements {
if _, err := session.ExecContext(context.Background(), stmt); err != nil {
failures = append(failures, fmt.Sprintf("statement %d: %v", n+1, err))
if !w.options.ContinueOnError {
break
}
}
}
if len(failures) > 0 {
return fmt.Errorf("failed to execute SQL: %s", strings.Join(failures, "; "))
} }
return nil return nil
} }
func stripComments(sqlText string) string {
lines := strings.Split(sqlText, "\n")
kept := lines[:0]
for _, line := range lines {
if !strings.HasPrefix(strings.TrimSpace(line), "--") {
kept = append(kept, line)
}
}
return strings.TrimSpace(strings.Join(kept, "\n"))
}
func quote(s string) string { return "`" + strings.ReplaceAll(s, "`", "``") + "`" } func quote(s string) string { return "`" + strings.ReplaceAll(s, "`", "``") + "`" }
func quoted(xs []string) string { func quoted(xs []string) string {
out := make([]string, len(xs)) out := make([]string, len(xs))
@@ -207,3 +434,21 @@ func quoted(xs []string) string {
} }
return strings.Join(out, ", ") return strings.Join(out, ", ")
} }
func validAction(action string) bool {
switch strings.ToUpper(action) {
case "", "CASCADE", "SET NULL", "RESTRICT", "NO ACTION":
return true
}
return false
}
func unsupportedMode(mode string) string {
var modes []string
for _, unsupported := range []string{"NO_BACKSLASH_ESCAPES", "ANSI_QUOTES"} {
if strings.Contains(strings.ToUpper(mode), unsupported) {
modes = append(modes, unsupported)
}
}
return strings.Join(modes, ", ")
}
+23 -7
View File
@@ -9,13 +9,6 @@ import (
"git.warky.dev/wdevs/relspecgo/pkg/writers" "git.warky.dev/wdevs/relspecgo/pkg/writers"
) )
func TestStripComments(t *testing.T) {
got := stripComments("-- header\nCREATE TABLE `users` (\n `id` INT\n);")
if strings.HasPrefix(got, "--") || !strings.HasPrefix(got, "CREATE TABLE") {
t.Fatalf("stripComments() = %q", got)
}
}
func TestWriterGeneratesMySQLDDL(t *testing.T) { func TestWriterGeneratesMySQLDDL(t *testing.T) {
db := models.InitDatabase("app") db := models.InitDatabase("app")
s := models.InitSchema("app") s := models.InitSchema("app")
@@ -42,3 +35,26 @@ func TestWriterGeneratesMySQLDDL(t *testing.T) {
} }
} }
} }
func TestDatabaseStatementsKeepEmbeddedDelimiters(t *testing.T) {
db := models.InitDatabase("app")
s := models.InitSchema("app")
db.Schemas = append(db.Schemas, s)
for _, name := range []string{"one", "two"} {
table := models.InitTable(name, s.Name)
col := models.InitColumn("text", name, s.Name)
col.Type = "text"
col.Default = "first;\n-- data, not a comment\nlast"
table.Columns[col.Name] = col
s.Tables = append(s.Tables, table)
}
statements := databaseStatements(db)
if len(statements) != 2 {
t.Fatalf("got %d statements", len(statements))
}
for _, stmt := range statements {
if !strings.Contains(stmt, "first;\n-- data, not a comment\nlast") {
t.Fatalf("default corrupted: %s", stmt)
}
}
}
+86
View File
@@ -17,6 +17,28 @@ The PostgreSQL Writer converts RelSpec's internal database model representation
- Includes audit triggers (optional) - Includes audit triggers (optional)
- Handles PostgreSQL-specific data types - Handles PostgreSQL-specific data types
## Identifier and value handling
Generated identifiers are always double-quoted, including reserved words, and
retain the model's case by default. Supply raw model names, without SQL quoting.
Identifiers containing double quotes are escaped by doubling them; NUL is rejected
before database/file output. Catalog checks, comments, audit values, and regclass
names use SQL literal quoting; generated procedural/function bodies choose dollar
delimiters absent from their rendered content. Names in generated line comments
cannot create new SQL lines.
Use CLI `--lowercase`, job option `lowercase_identifiers: true`, or PostgreSQL
`WriterOptions.LowercaseIdentifiers: true` to lowercase model names and structured
references. The transformation copies the model and rejects name collisions. Other
library writers can use `transform.LowercaseIdentifiers` before writing.
SQL expressions are executable input: types, defaults, generated expressions,
checks, index expressions/predicates, dialect directive clauses, audit user
expressions, and `pgsql_using` / `pgsql_preflight` are not sanitized or case-folded.
Adjust identifier references inside these expressions yourself when lowercasing.
Enum values, comments, and other data retain their case. Migration matching respects
identifier case; `"Users"` and `"users"` are distinct objects.
## Usage ## Usage
### Basic Example ### Basic Example
@@ -346,3 +368,67 @@ metadata:
- Default values are properly quoted - Default values are properly quoted
- Constraint names follow PostgreSQL conventions - Constraint names follow PostgreSQL conventions
- Compatible with PostgreSQL 12+ - Compatible with PostgreSQL 12+
## Live execution failures
Live PostgreSQL output stops after the first failed statement by default and returns
`*writers.ExecutionFailure` (also available as `*pgsql.ExecutionFailure`; inspect it
with `errors.As`). `WriterOptions.ContinueOnError` or job
`options.continue_on_error: true` attempts later statements but still returns an error
if any fail, preventing dependent jobs from running. A failed type preflight always
stops live execution, including when continuation is enabled.
PostgreSQL, SQLite, SQL Server, and the PostgreSQL script runner share execution
reports through `WriterOptions.Metadata["execution_report"]`. Optional
`Metadata["report_path"]` writes the report as JSON; write and close failures are
errors. Reports include each operation's status and original number, plus applied,
failed, skipped, and rolled-back counts. Blank and comment entries count as skipped.
Unknown commit/rollback outcomes are explicitly marked `unknown` and are never
reported as confirmed rollback. Connection failures record all planned operations as
skipped. Raw scripts have different reporting limits; see the sqlexec README.
Generated DDL uses independent statements by default: earlier changes can remain
committed after failure. Set `WriterOptions.AtomicExecution = true` to use one
transaction with PostgreSQL, SQLite, or SQL Server generated DDL. For database output
jobs, use:
```yaml
options:
atomic_execution: true
```
Atomic execution and continuation are mutually exclusive. Rollback covers database
transactional effects; sequence advancement and external effects in SQL expressions
are outside that guarantee. Concurrency locks, migration history, and resumable
migration management remain separate work (audit A09).
## Column type conversions
Type changes propagate errors and never implicitly rename or replace a column.
Before schema changes, full-DDL and diff plans evaluate every existing value through
the conversion expression and target cast. Diff preflights run before dropping
constraints/indexes or creating sequences; database-wide preflights run before
changing any schema. Matching types, missing tables/columns, generated columns, and
identity columns need no conversion check.
Column metadata in JSON/YAML models supports two optional SQL expressions:
```yaml
metadata:
pgsql_using: "split_part(name, ' ', 1)::integer"
pgsql_preflight: "name ~ '^[0-9]+ units$'"
```
`pgsql_using` replaces the default `column::target_type` expression in `USING`.
`pgsql_preflight` is a row predicate; false or NULL rejects the conversion. These
fields must contain nonempty strings. Both are trusted executable SQL, like defaults
and check expressions; they are not values or a sandbox. Preflights evaluate the
conversion expression, so use expressions without side effects. New types/extensions
needed by an existing-column conversion must already be available for preflight.
Preflight checks can race concurrent writes and do not prove that locks, defaults,
or every dependency will permit the ALTER. The ALTER still validates the conversion;
use atomic execution when preceding operations also need rollback. Generated SQL
clients must stop on preflight failure; live RelSpec execution enforces this even
with continuation enabled. Rename/replacement/backfill operations still require a
separately reviewed migration.
+5 -4
View File
@@ -22,8 +22,9 @@ func TestCurrentColumnHasDescription(t *testing.T) {
want bool want bool
}{ }{
{"nil table", nil, &models.Column{Name: "email", Description: "x"}, false}, {"nil table", nil, &models.Column{Name: "email", Description: "x"}, false},
{"match ignoring case and whitespace", table, &models.Column{Name: "email", Description: "the email"}, true}, {"match ignoring whitespace", table, &models.Column{Name: "Email", Description: "the email"}, true},
{"different description", table, &models.Column{Name: "email", Description: "other"}, false}, {"different description", table, &models.Column{Name: "Email", Description: "other"}, false},
{"different identifier case", table, &models.Column{Name: "email", Description: "the email"}, false},
{"column missing", table, &models.Column{Name: "age", Description: "x"}, false}, {"column missing", table, &models.Column{Name: "age", Description: "x"}, false},
} }
for _, tt := range tests { for _, tt := range tests {
@@ -41,12 +42,12 @@ func TestExecuteCommentColumn(t *testing.T) {
t.Fatal(err) t.Fatal(err)
} }
got, err := te.ExecuteCommentColumn(CommentColumnData{ got, err := te.ExecuteCommentColumn(CommentColumnData{
SchemaName: "public", TableName: "users", ColumnName: "email", Comment: "it''s", SchemaName: "public", TableName: "users", ColumnName: "email", Comment: "it's",
}) })
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
if !strings.Contains(got, "COMMENT ON COLUMN") || !strings.Contains(got, "public.users") || if !strings.Contains(got, "COMMENT ON COLUMN") || !strings.Contains(got, "\"public\".\"users\"") ||
!strings.Contains(got, "email") || !strings.Contains(got, "IS 'it''s';") { !strings.Contains(got, "email") || !strings.Contains(got, "IS 'it''s';") {
t.Errorf("unexpected output: %s", got) t.Errorf("unexpected output: %s", got)
} }
+11 -11
View File
@@ -57,8 +57,8 @@ func TestDiffStatements(t *testing.T) {
current: func() *models.Database { current: func() *models.Database {
return diffTestDB("public", map[string]map[string]string{"users": {"id": "integer"}}) return diffTestDB("public", map[string]map[string]string{"users": {"id": "integer"}})
}, },
wantContain: []string{"CREATE TABLE IF NOT EXISTS public.posts"}, wantContain: []string{"CREATE TABLE IF NOT EXISTS \"public\".\"posts\""},
wantAbsent: []string{"public.users"}, wantAbsent: []string{"\"public\".\"users\""},
}, },
{ {
name: "missing column is added and existing columns are skipped", name: "missing column is added and existing columns are skipped",
@@ -68,7 +68,7 @@ func TestDiffStatements(t *testing.T) {
current: func() *models.Database { current: func() *models.Database {
return diffTestDB("public", map[string]map[string]string{"users": {"id": "integer"}}) return diffTestDB("public", map[string]map[string]string{"users": {"id": "integer"}})
}, },
wantContain: []string{"ADD COLUMN IF NOT EXISTS email text"}, wantContain: []string{"ADD COLUMN IF NOT EXISTS \"email\" text"},
wantAbsent: []string{"COLUMN id"}, wantAbsent: []string{"COLUMN id"},
}, },
{ {
@@ -79,7 +79,7 @@ func TestDiffStatements(t *testing.T) {
current: func() *models.Database { current: func() *models.Database {
return diffTestDB("public", map[string]map[string]string{"users": {"id": "integer"}}) return diffTestDB("public", map[string]map[string]string{"users": {"id": "integer"}})
}, },
wantContain: []string{"ALTER COLUMN id TYPE bigint"}, wantContain: []string{"ALTER COLUMN \"id\" TYPE bigint"},
}, },
{ {
name: "missing non-public schema is created before its tables", name: "missing non-public schema is created before its tables",
@@ -88,8 +88,8 @@ func TestDiffStatements(t *testing.T) {
}, },
current: func() *models.Database { return models.InitDatabase("testdb") }, current: func() *models.Database { return models.InitDatabase("testdb") },
wantContain: []string{ wantContain: []string{
"CREATE SCHEMA IF NOT EXISTS app", "CREATE SCHEMA IF NOT EXISTS \"app\"",
"CREATE TABLE IF NOT EXISTS app.users", "CREATE TABLE IF NOT EXISTS \"app\".\"users\"",
}, },
}, },
} }
@@ -149,7 +149,7 @@ func TestDiffStatements_NewTableCreatesPrimaryKeySequence(t *testing.T) {
id := models.InitColumn("id", "users", "public") id := models.InitColumn("id", "users", "public")
id.Type = "integer" id.Type = "integer"
id.IsPrimaryKey = true id.IsPrimaryKey = true
id.Default = "nextval('public.users_id_seq'::regclass)" id.Default = "nextval('\"public\".\"users_id_seq\"'::regclass)"
table.Columns["id"] = id table.Columns["id"] = id
schema.Tables = append(schema.Tables, table) schema.Tables = append(schema.Tables, table)
model.Schemas = append(model.Schemas, schema) model.Schemas = append(model.Schemas, schema)
@@ -160,7 +160,7 @@ func TestDiffStatements_NewTableCreatesPrimaryKeySequence(t *testing.T) {
t.Fatalf("diffStatements failed: %v", err) t.Fatalf("diffStatements failed: %v", err)
} }
out := diffJoin(stmts) out := diffJoin(stmts)
if !strings.Contains(out, "CREATE SEQUENCE IF NOT EXISTS public.users_id_seq") { if !strings.Contains(out, "CREATE SEQUENCE IF NOT EXISTS \"public\".\"users_id_seq\"") {
t.Fatalf("expected sequence creation, got:\n%s", out) t.Fatalf("expected sequence creation, got:\n%s", out)
} }
@@ -193,7 +193,7 @@ func TestDiffStatements_NewIndexAndChangedIndexRecreated(t *testing.T) {
if err != nil { if err != nil {
t.Fatalf("diffStatements failed: %v", err) t.Fatalf("diffStatements failed: %v", err)
} }
if !strings.Contains(diffJoin(stmts), "CREATE INDEX IF NOT EXISTS idx_users_lookup") { if !strings.Contains(diffJoin(stmts), "CREATE INDEX IF NOT EXISTS \"idx_users_lookup\"") {
t.Fatalf("expected index creation, got:\n%s", diffJoin(stmts)) t.Fatalf("expected index creation, got:\n%s", diffJoin(stmts))
} }
@@ -329,7 +329,7 @@ func TestDiffStatements_ChangedPrimaryKeyIsRecreated(t *testing.T) {
if err != nil { if err != nil {
t.Fatalf("diffStatements failed: %v", err) t.Fatalf("diffStatements failed: %v", err)
} }
if !strings.Contains(diffJoin(stmts), "DROP CONSTRAINT IF EXISTS pk_public_users") { if !strings.Contains(diffJoin(stmts), "DROP CONSTRAINT IF EXISTS \"pk_public_users\"") {
t.Fatalf("expected old primary key to be dropped, got:\n%s", diffJoin(stmts)) t.Fatalf("expected old primary key to be dropped, got:\n%s", diffJoin(stmts))
} }
} }
@@ -345,7 +345,7 @@ func TestDiffStatements_ChangedCommentEmitted(t *testing.T) {
if err != nil { if err != nil {
t.Fatalf("diffStatements failed: %v", err) t.Fatalf("diffStatements failed: %v", err)
} }
if !strings.Contains(diffJoin(stmts), "COMMENT ON TABLE public.users IS 'new'") { if !strings.Contains(diffJoin(stmts), "COMMENT ON TABLE \"public\".\"users\" IS 'new'") {
t.Fatalf("expected changed comment, got:\n%s", diffJoin(stmts)) t.Fatalf("expected changed comment, got:\n%s", diffJoin(stmts))
} }
} }
+2 -2
View File
@@ -55,7 +55,7 @@ func TestPgDirectives_WriteDatabasePath(t *testing.T) {
for _, want := range []string{ for _, want := range []string{
") PARTITION BY RANGE (created_at) TABLESPACE fast_data", ") PARTITION BY RANGE (created_at) TABLESPACE fast_data",
"GENERATED ALWAYS AS IDENTITY", "GENERATED ALWAYS AS IDENTITY",
"WITH (fillfactor=90) TABLESPACE idx_space", "WITH (fillfactor=90) TABLESPACE \"idx_space\"",
} { } {
if !strings.Contains(out, want) { if !strings.Contains(out, want) {
t.Errorf("missing %q in:\n%s", want, out) t.Errorf("missing %q in:\n%s", want, out)
@@ -78,7 +78,7 @@ func TestPgDirectives_WriteSchemaPath(t *testing.T) {
if !strings.Contains(out, ") PARTITION BY RANGE (created_at) TABLESPACE fast_data;") { if !strings.Contains(out, ") PARTITION BY RANGE (created_at) TABLESPACE fast_data;") {
t.Errorf("table suffix missing from WriteSchema path:\n%s", out) t.Errorf("table suffix missing from WriteSchema path:\n%s", out)
} }
if !strings.Contains(out, "WITH (fillfactor=90) TABLESPACE idx_space") { if !strings.Contains(out, "WITH (fillfactor=90) TABLESPACE \"idx_space\"") {
t.Errorf("index clauses missing from WriteSchema path:\n%s", out) t.Errorf("index clauses missing from WriteSchema path:\n%s", out)
} }
} }
+2 -2
View File
@@ -88,8 +88,8 @@ func TestGeneratedColumns_FullDDL(t *testing.T) {
} }
out := diffJoin(stmts) out := diffJoin(stmts)
for _, s := range []string{ for _, s := range []string{
"full_name text GENERATED ALWAYS AS (first || ' ' || last) STORED", "\"full_name\" text GENERATED ALWAYS AS (first || ' ' || last) STORED",
"seq bigint GENERATED ALWAYS AS IDENTITY NOT NULL", "\"seq\" bigint GENERATED ALWAYS AS IDENTITY NOT NULL",
} { } {
if !strings.Contains(out, s) { if !strings.Contains(out, s) {
t.Errorf("missing %q:\n%s", s, out) t.Errorf("missing %q:\n%s", s, out)
+116
View File
@@ -0,0 +1,116 @@
package pgsql
import (
"fmt"
"reflect"
"strings"
"git.warky.dev/wdevs/relspecgo/pkg/models"
"git.warky.dev/wdevs/relspecgo/pkg/transform"
"git.warky.dev/wdevs/relspecgo/pkg/writers"
)
// prepareModel rejects NUL before creating files or executing SQL. SQL names
// cannot contain NUL; pgx's identifier sanitizer would otherwise remove it.
func prepareModel(db *models.Database, lower bool) (*models.Database, error) {
if db == nil {
return nil, fmt.Errorf("database is required")
}
seen := map[any]bool{}
var validate func(reflect.Value) error
validate = func(v reflect.Value) error {
switch v.Kind() {
case reflect.Pointer:
if v.IsNil() || v.Type().Elem().PkgPath() != reflect.TypeOf(models.Database{}).PkgPath() {
return nil
}
if seen[v.Interface()] {
return nil
}
seen[v.Interface()] = true
return validate(v.Elem())
case reflect.Struct:
for i := 0; i < v.NumField(); i++ {
if v.Type().Field(i).Name == "Metadata" {
continue
}
if err := validate(v.Field(i)); err != nil {
return err
}
}
case reflect.String:
if strings.ContainsRune(v.String(), 0) {
return fmt.Errorf("PostgreSQL model contains a NUL byte")
}
case reflect.Slice:
for i := 0; i < v.Len(); i++ {
if err := validate(v.Index(i)); err != nil {
return err
}
}
case reflect.Map:
if v.Type().Elem().Kind() == reflect.Pointer {
iter := v.MapRange()
for iter.Next() {
if err := validate(iter.Value()); err != nil {
return err
}
}
}
}
return nil
}
if err := validate(reflect.ValueOf(db)); err != nil {
return nil, err
}
if lower {
return transform.LowercaseIdentifiers(db)
}
return db, nil
}
func (w *Writer) prepareSchema(schema *models.Schema) (*models.Schema, error) {
db, err := prepareModel(&models.Database{Schemas: []*models.Schema{schema}}, w.options.LowercaseIdentifiers)
if err != nil {
return nil, err
}
return db.Schemas[0], nil
}
// regclassLastName decodes quoted identifiers in a qualified regclass spelling.
func regclassLastName(text string) string {
var name strings.Builder
quoted := false
for i := 0; i < len(text); i++ {
c := text[i]
if c == '"' {
if quoted && i+1 < len(text) && text[i+1] == '"' {
name.WriteByte('"')
i++
} else {
quoted = !quoted
}
continue
}
if c == '.' && !quoted {
name.Reset()
continue
}
name.WriteByte(c)
}
return name.String()
}
// quoteDefaultValue keeps explicit SQL expressions intact and uses the same
// backslash-safe literal quoting as other PostgreSQL value contexts.
func quoteDefaultValue(value, sqlType string) string {
trimmed := strings.TrimSpace(value)
if len(trimmed) >= 3 && (trimmed[0] == 'E' || trimmed[0] == 'e') && trimmed[1] == '\'' && trimmed[len(trimmed)-1] == '\'' {
return trimmed
}
formatted := writers.QuoteDefaultValue(value, sqlType)
if formatted == "'"+strings.ReplaceAll(trimmed, "'", "''")+"'" {
return quote(trimmed)
}
return formatted
}
+178 -4
View File
@@ -3,9 +3,11 @@ package pgsql
import ( import (
"context" "context"
"encoding/json" "encoding/json"
"errors"
"fmt" "fmt"
"os" "os"
"path/filepath" "path/filepath"
"strings"
"testing" "testing"
"time" "time"
@@ -141,15 +143,16 @@ func TestLive_WriteDatabaseEmptyThenIdenticalThenDrifted(t *testing.T) {
} }
} }
func TestLive_WriteDatabaseFailedStatementContinues(t *testing.T) { func TestLive_WriteDatabaseFailedStatementReturnsError(t *testing.T) {
connString := liveWriterConn(t) connString := liveWriterConn(t)
schema, _ := liveWriterSchema(t, connString) schema, _ := liveWriterSchema(t, connString)
reportPath := filepath.Join(t.TempDir(), "report.json") reportPath := filepath.Join(t.TempDir(), "report.json")
db := liveModel(schema, map[string]string{"bad": "no_such_type_xyz"}) db := liveModel(schema, map[string]string{"bad": "no_such_type_xyz"})
rep, err := runLiveWrite(t, connString, db, map[string]interface{}{"full_ddl": true, "report_path": reportPath}) rep, err := runLiveWrite(t, connString, db, map[string]interface{}{"full_ddl": true, "report_path": reportPath})
if err != nil { var failure *ExecutionFailure
t.Fatalf("failed statements must not abort the run: %v", err) if !errors.As(err, &failure) {
t.Fatalf("expected structured failure, got: %v", err)
} }
if rep.FailedStatements == 0 || len(rep.Errors) != rep.FailedStatements { if rep.FailedStatements == 0 || len(rep.Errors) != rep.FailedStatements {
t.Fatalf("expected recorded failures: %+v", rep) t.Fatalf("expected recorded failures: %+v", rep)
@@ -230,7 +233,7 @@ func TestLive_ExecuteStatementsSkipsCommentsAndBlank(t *testing.T) {
t.Fatal(err) t.Fatal(err)
} }
r := w.executionReport r := w.executionReport
if r.ExecutedStatements != 2 || r.FailedStatements != 0 || r.TotalStatements != 5 { if r.ExecutedStatements != 2 || r.FailedStatements != 0 || r.TotalStatements != 5 || r.SkippedStatements != 3 {
t.Errorf("counts: %+v", r) t.Errorf("counts: %+v", r)
} }
if len(r.Schemas) != 1 || r.Schemas[0].Name != schema || len(r.Schemas[0].Tables) != 1 || !r.Schemas[0].Tables[0].Created { if len(r.Schemas) != 1 || r.Schemas[0].Name != schema || len(r.Schemas[0].Tables) != 1 || !r.Schemas[0].Tables[0].Created {
@@ -241,3 +244,174 @@ func TestLive_ExecuteStatementsSkipsCommentsAndBlank(t *testing.T) {
t.Errorf("table not created: %v", err) t.Errorf("table not created: %v", err)
} }
} }
func TestLive_ExecuteStatementsFailureModes(t *testing.T) {
connString := liveWriterConn(t)
for _, continueOnError := range []bool{false, true} {
t.Run(fmt.Sprintf("continue=%v", continueOnError), func(t *testing.T) {
schema, conn := liveWriterSchema(t, connString)
reportPath := filepath.Join(t.TempDir(), "report.json")
w := NewWriter(&writers.WriterOptions{
ContinueOnError: continueOnError,
Metadata: map[string]interface{}{"report_path": reportPath},
})
w.executionReport = &ExecutionReport{}
err := w.executeStatements([]string{
"CREATE SCHEMA " + schema,
"SELECT 1 / 0",
"CREATE TABLE " + schema + ".after_failure (id int)",
}, connString)
var failure *ExecutionFailure
if !errors.As(err, &failure) || failure.Report != w.executionReport {
t.Fatalf("expected structured execution failure, got %v", err)
}
var report ExecutionReport
data, err := os.ReadFile(reportPath)
if err != nil {
t.Fatal(err)
}
if err := json.Unmarshal(data, &report); err != nil {
t.Fatal(err)
}
executed, skipped := 1, 1
if continueOnError {
executed, skipped = 2, 0
}
if report.TotalStatements != 3 || report.ExecutedStatements != executed || report.FailedStatements != 1 || report.SkippedStatements != skipped || report.EndTime == "" {
t.Fatalf("incorrect partial application report: %+v", report)
}
if len(report.Errors) != 1 || report.Errors[0].StatementNumber != 2 {
t.Fatalf("incorrect failure details: %+v", report.Errors)
}
if exists := columnExists(t, conn, schema, "after_failure", "id"); exists != continueOnError {
t.Fatalf("later statement executed=%v, continue=%v", exists, continueOnError)
}
})
}
}
func TestLive_IncompatibleTypeChangePreservesColumnAndData(t *testing.T) {
connString := liveWriterConn(t)
schema, conn := liveWriterSchema(t, connString)
ctx := context.Background()
_, err := conn.Exec(ctx, "CREATE SCHEMA "+schema+"; CREATE TABLE "+schema+".accounts (id integer PRIMARY KEY, name text UNIQUE); INSERT INTO "+schema+".accounts VALUES (1, 'not-an-integer')")
if err != nil {
t.Fatal(err)
}
var originalIndex uint32
if err := conn.QueryRow(ctx, "SELECT to_regclass($1)::oid", schema+".accounts_name_key").Scan(&originalIndex); err != nil {
t.Fatal(err)
}
// Exercise both full-DDL and diff generation against populated data.
for _, fullDDL := range []bool{false, true} {
t.Run(fmt.Sprintf("full_ddl=%v", fullDDL), func(t *testing.T) {
model := liveModel(schema, map[string]string{"name": "integer"})
unique := models.InitConstraint("accounts_name_key", models.UniqueConstraint)
unique.Columns = []string{"name"}
model.Schemas[0].Tables[0].Constraints[unique.Name] = unique
report, err := runLiveWrite(t, connString, model, map[string]interface{}{"full_ddl": fullDDL})
var failure *ExecutionFailure
if !errors.As(err, &failure) || report.FailedStatements != 1 {
t.Fatalf("expected failed conversion, got report=%+v err=%v", report, err)
}
var value, typ string
if err := conn.QueryRow(ctx, "SELECT name, pg_typeof(name)::text FROM "+schema+".accounts WHERE id=1").Scan(&value, &typ); err != nil {
t.Fatal(err)
}
if value != "not-an-integer" || typ != "text" {
t.Fatalf("original column changed: value=%q type=%q", value, typ)
}
var columns int
if err := conn.QueryRow(ctx, "SELECT count(*) FROM information_schema.columns WHERE table_schema=$1 AND table_name='accounts'", schema).Scan(&columns); err != nil || columns != 2 {
t.Fatalf("unexpected replacement column: columns=%d err=%v", columns, err)
}
var index uint32
if err := conn.QueryRow(ctx, "SELECT to_regclass($1)::oid", schema+".accounts_name_key").Scan(&index); err != nil || index != originalIndex {
t.Fatalf("unique index changed: index=%d original=%d err=%v", index, originalIndex, err)
}
if _, err := conn.Exec(ctx, "INSERT INTO "+schema+".accounts VALUES (2, 'not-an-integer')"); err == nil {
t.Fatal("unique constraint no longer enforced")
}
})
}
}
func TestLive_AtomicExecutionRollback(t *testing.T) {
connString := liveWriterConn(t)
schema, conn := liveWriterSchema(t, connString)
w := NewWriter(&writers.WriterOptions{AtomicExecution: true})
w.executionReport = &ExecutionReport{}
err := w.executeStatements([]string{
"-- Schema: " + schema,
"CREATE SCHEMA " + schema,
"CREATE TABLE " + schema + ".created (id int)",
"SELECT 1 / 0",
"SELECT 1",
}, connString)
var failure *ExecutionFailure
if !errors.As(err, &failure) {
t.Fatalf("expected execution failure: %v", err)
}
report := failure.Report
if report.Outcome != "rolled_back" || report.RolledBackStatements != 2 || report.ExecutedStatements != 0 || report.FailedStatements != 1 || report.SkippedStatements != 2 {
t.Fatalf("rollback report: %+v", report)
}
if report.Schemas[0].Tables[0].Created {
t.Fatal("rolled-back table must not be reported created")
}
if columnExists(t, conn, schema, "created", "id") {
t.Fatal("table survived transaction rollback")
}
}
func TestLive_TypePreflightPrecedesDropsEvenWithContinuation(t *testing.T) {
connString := liveWriterConn(t)
schema, conn := liveWriterSchema(t, connString)
_, err := conn.Exec(context.Background(), "CREATE SCHEMA "+schema+"; CREATE TABLE "+schema+".accounts (id integer PRIMARY KEY, name text UNIQUE); INSERT INTO "+schema+".accounts VALUES (1, '123'), (2, 'bad')")
if err != nil {
t.Fatal(err)
}
// The desired model would drop the unique constraint. Incompatible data must
// be caught first, including bad data after the first row.
model := liveModel(schema, map[string]string{"name": "integer"})
w := NewWriter(&writers.WriterOptions{ContinueOnError: true, Metadata: map[string]interface{}{"connection_string": connString}})
err = w.WriteDatabase(model)
var failure *ExecutionFailure
if !errors.As(err, &failure) || failure.Report.FailedStatements != 1 || failure.Report.ExecutedStatements != 0 {
t.Fatalf("preflight must stop mutations: %+v %v", w.executionReport, err)
}
var exists bool
if err := conn.QueryRow(context.Background(), "SELECT to_regclass($1) IS NOT NULL", schema+".accounts_name_key").Scan(&exists); err != nil || !exists {
t.Fatalf("constraint was dropped before preflight: %v", err)
}
}
func TestLive_CustomTypeConversionAndPredicate(t *testing.T) {
connString := liveWriterConn(t)
for _, fullDDL := range []bool{false, true} {
t.Run(fmt.Sprintf("full_ddl=%v", fullDDL), func(t *testing.T) {
schema, conn := liveWriterSchema(t, connString)
_, err := conn.Exec(context.Background(), "CREATE SCHEMA "+schema+"; CREATE TABLE "+schema+".accounts (id integer PRIMARY KEY, name text); INSERT INTO "+schema+".accounts VALUES (1,'42 units'),(2,'7 units')")
if err != nil {
t.Fatal(err)
}
model := liveModel(schema, map[string]string{"name": "integer"})
col := model.Schemas[0].Tables[0].Columns["name"]
col.Metadata["pgsql_using"] = "split_part(name, ' ', 1)::integer"
col.Metadata["pgsql_preflight"] = "name ~ '^[0-9]+ units$' AND name NOT LIKE '7 %'"
meta := map[string]interface{}{"full_ddl": fullDDL}
if _, err := runLiveWrite(t, connString, model, meta); err == nil || !strings.Contains(err.Error(), "preflight rejected rows") {
t.Fatalf("predicate must reject: %v", err)
}
col.Metadata["pgsql_preflight"] = "name ~ '^[0-9]+ units$'"
if _, err := runLiveWrite(t, connString, model, meta); err != nil {
t.Fatal(err)
}
var value int
if err := conn.QueryRow(context.Background(), "SELECT sum(name) FROM "+schema+".accounts").Scan(&value); err != nil || value != 49 {
t.Fatalf("custom conversion result %d: %v", value, err)
}
})
}
}
+54 -43
View File
@@ -61,6 +61,16 @@ func (w *MigrationWriter) GenerateScripts(model, current *models.Database) ([]Mi
current = models.InitDatabase(model.Name) current = models.InitDatabase(model.Name)
} }
var err error
model, err = prepareModel(model, w.options.LowercaseIdentifiers)
if err != nil {
return nil, err
}
current, err = prepareModel(current, false)
if err != nil {
return nil, err
}
// Check if audit is configured in metadata // Check if audit is configured in metadata
var auditConfig *AuditConfig var auditConfig *AuditConfig
if w.options.Metadata != nil { if w.options.Metadata != nil {
@@ -93,7 +103,7 @@ func (w *MigrationWriter) GenerateScripts(model, current *models.Database) ([]Mi
if cs == nil { if cs == nil {
continue continue
} }
if strings.EqualFold(cs.Name, modelSchema.Name) { if cs.Name == modelSchema.Name {
currentSchema = cs currentSchema = cs
break break
} }
@@ -166,7 +176,7 @@ func (w *MigrationWriter) WriteMigration(model, current *models.Database) error
// Write header // Write header
fmt.Fprintf(w.writer, "-- PostgreSQL Migration Script\n") fmt.Fprintf(w.writer, "-- PostgreSQL Migration Script\n")
fmt.Fprintf(w.writer, "-- Generated by RelSpec\n") fmt.Fprintf(w.writer, "-- Generated by RelSpec\n")
fmt.Fprintf(w.writer, "-- Source: %s -> %s\n", current.Name, model.Name) fmt.Fprintf(w.writer, "-- Source: %s -> %s\n", commentText(current.Name), commentText(model.Name))
if w.options.ContinueOnError { if w.options.ContinueOnError {
fmt.Fprintf(w.writer, "\\set ON_ERROR_STOP off\n") fmt.Fprintf(w.writer, "\\set ON_ERROR_STOP off\n")
} }
@@ -175,7 +185,7 @@ func (w *MigrationWriter) WriteMigration(model, current *models.Database) error
// Write scripts // Write scripts
for _, script := range scripts { for _, script := range scripts {
fmt.Fprintf(w.writer, "-- Priority: %d | Type: %s | Object: %s\n", fmt.Fprintf(w.writer, "-- Priority: %d | Type: %s | Object: %s\n",
script.Priority, script.ObjectType, script.ObjectName) script.Priority, commentText(script.ObjectType), commentText(script.ObjectName))
fmt.Fprintf(w.writer, "%s\n\n", script.Body) fmt.Fprintf(w.writer, "%s\n\n", script.Body)
} }
@@ -256,12 +266,12 @@ func (w *MigrationWriter) generateDropScripts(model, current *models.Schema) ([]
// Build map of model tables for quick lookup // Build map of model tables for quick lookup
modelTables := make(map[string]*models.Table) modelTables := make(map[string]*models.Table)
for _, table := range model.Tables { for _, table := range model.Tables {
modelTables[strings.ToLower(table.Name)] = table modelTables[table.Name] = table
} }
// Find constraints to drop // Find constraints to drop
for _, currentTable := range current.Tables { for _, currentTable := range current.Tables {
modelTable, existsInModel := modelTables[strings.ToLower(currentTable.Name)] modelTable, existsInModel := modelTables[currentTable.Name]
if !existsInModel { if !existsInModel {
continue continue
@@ -296,7 +306,7 @@ func (w *MigrationWriter) generateDropScripts(model, current *models.Schema) ([]
if refSchema == "" { if refSchema == "" {
refSchema = current.Name refSchema = current.Name
} }
if strings.EqualFold(refTable, currentTable.Name) && strings.EqualFold(refSchema, current.Name) { if (refTable == currentTable.Name) && (refSchema == current.Name) {
fkKey := fmt.Sprintf("%s.%s.%s", current.Name, otherTable.Name, fkName) fkKey := fmt.Sprintf("%s.%s.%s", current.Name, otherTable.Name, fkName)
if !droppedFKs[fkKey] { if !droppedFKs[fkKey] {
droppedFKs[fkKey] = true droppedFKs[fkKey] = true
@@ -395,13 +405,13 @@ func (w *MigrationWriter) generateTableScripts(model, current *models.Schema) ([
currentTables := make(map[string]*models.Table) currentTables := make(map[string]*models.Table)
if current != nil { if current != nil {
for _, table := range current.Tables { for _, table := range current.Tables {
currentTables[strings.ToLower(table.Name)] = table currentTables[table.Name] = table
} }
} }
// Process each model table // Process each model table
for _, modelTable := range model.Tables { for _, modelTable := range model.Tables {
currentTable, exists := currentTables[strings.ToLower(modelTable.Name)] currentTable, exists := currentTables[modelTable.Name]
if !exists { if !exists {
// Table doesn't exist, create it // Table doesn't exist, create it
@@ -439,12 +449,12 @@ func (w *MigrationWriter) generateAlterTableScripts(schema *models.Schema, model
// Build map of current columns // Build map of current columns
currentColumns := make(map[string]*models.Column) currentColumns := make(map[string]*models.Column)
for name, col := range currentTable.Columns { for name, col := range currentTable.Columns {
currentColumns[strings.ToLower(name)] = col currentColumns[name] = col
} }
// Check each model column // Check each model column
for _, modelCol := range sortColumns(modelTable.Columns) { for _, modelCol := range sortColumns(modelTable.Columns) {
currentCol, exists := currentColumns[strings.ToLower(modelCol.Name)] currentCol, exists := currentColumns[modelCol.Name]
if !exists { if !exists {
// Column doesn't exist, add it // Column doesn't exist, add it
@@ -487,15 +497,16 @@ func (w *MigrationWriter) generateAlterTableScripts(schema *models.Schema, model
} else if !columnsEqual(modelCol, currentCol) { } else if !columnsEqual(modelCol, currentCol) {
// Column exists but properties changed // Column exists but properties changed
if !columnTypesEqual(modelCol, currentCol) { if !columnTypesEqual(modelCol, currentCol) {
newType := effectiveAlterColumnSQLType(modelCol) data, err := conversionData(schema.Name, modelTable.Name, modelCol)
sql, err := w.executor.ExecuteAlterColumnTypeWithCheck(AlterColumnTypeWithCheckData{ if err != nil {
SchemaName: schema.Name, return nil, err
TableName: modelTable.Name, }
ColumnName: modelCol.Name, scripts = append(scripts, MigrationScript{
NewType: newType, ObjectName: fmt.Sprintf("%s.%s.%s", schema.Name, modelTable.Name, modelCol.Name),
EquivalentTypes: equivalentTypeListSQL(newType), ObjectType: "preflight column type", Schema: schema.Name, Priority: -10000,
UsingExpr: buildAlterColumnUsingExpression(modelCol.Name, newType), Sequence: len(scripts), Body: preflightColumnType(data, w.options.FlattenSchema),
}) })
sql, err := w.executor.ExecuteAlterColumnTypeWithCheck(data)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@@ -573,13 +584,13 @@ func (w *MigrationWriter) generateIndexScripts(model, current *models.Schema) ([
currentTables := make(map[string]*models.Table) currentTables := make(map[string]*models.Table)
if current != nil { if current != nil {
for _, table := range current.Tables { for _, table := range current.Tables {
currentTables[strings.ToLower(table.Name)] = table currentTables[table.Name] = table
} }
} }
// Process each model table // Process each model table
for _, modelTable := range model.Tables { for _, modelTable := range model.Tables {
currentTable := currentTables[strings.ToLower(modelTable.Name)] currentTable := currentTables[modelTable.Name]
// Process primary keys first - check explicit constraints // Process primary keys first - check explicit constraints
foundExplicitPK := false foundExplicitPK := false
@@ -602,7 +613,7 @@ func (w *MigrationWriter) generateIndexScripts(model, current *models.Schema) ([
SchemaName: model.Name, SchemaName: model.Name,
TableName: modelTable.Name, TableName: modelTable.Name,
ConstraintName: constraintName, ConstraintName: constraintName,
Columns: strings.Join(constraint.Columns, ", "), Columns: quotedIdentifiers(constraint.Columns),
}) })
if err != nil { if err != nil {
return nil, err return nil, err
@@ -626,12 +637,12 @@ func (w *MigrationWriter) generateIndexScripts(model, current *models.Schema) ([
pkColumns := []string{} pkColumns := []string{}
for _, col := range modelTable.Columns { for _, col := range modelTable.Columns {
if col.IsPrimaryKey { if col.IsPrimaryKey {
pkColumns = append(pkColumns, col.SQLName()) pkColumns = append(pkColumns, col.Name)
} }
} }
if len(pkColumns) > 0 { if len(pkColumns) > 0 {
sort.Strings(pkColumns) sort.Strings(pkColumns)
constraintName := fmt.Sprintf("pk_%s_%s", model.SQLName(), modelTable.SQLName()) constraintName := fmt.Sprintf("pk_%s_%s", model.Name, modelTable.Name)
shouldCreate := true shouldCreate := true
if currentTable != nil { if currentTable != nil {
@@ -649,7 +660,7 @@ func (w *MigrationWriter) generateIndexScripts(model, current *models.Schema) ([
SchemaName: model.Name, SchemaName: model.Name,
TableName: modelTable.Name, TableName: modelTable.Name,
ConstraintName: constraintName, ConstraintName: constraintName,
Columns: strings.Join(pkColumns, ", "), Columns: quotedIdentifiers(pkColumns),
}) })
if err != nil { if err != nil {
return nil, err return nil, err
@@ -749,7 +760,7 @@ func buildIndexColumnExpressionsFiltered(table *models.Table, index *models.Inde
continue continue
} }
colExpr := col.SQLName() colExpr := quoteIdent(col.Name)
if opClass := indexOperatorClassForColumn(col, indexType, index.Comment); opClass != "" { if opClass := indexOperatorClassForColumn(col, indexType, index.Comment); opClass != "" {
colExpr = fmt.Sprintf("%s %s", colExpr, opClass) colExpr = fmt.Sprintf("%s %s", colExpr, opClass)
} }
@@ -766,12 +777,12 @@ func (w *MigrationWriter) generateUniqueConstraintScripts(model, current *models
currentTables := make(map[string]*models.Table) currentTables := make(map[string]*models.Table)
if current != nil { if current != nil {
for _, table := range current.Tables { for _, table := range current.Tables {
currentTables[strings.ToLower(table.Name)] = table currentTables[table.Name] = table
} }
} }
for _, modelTable := range model.Tables { for _, modelTable := range model.Tables {
currentTable := currentTables[strings.ToLower(modelTable.Name)] currentTable := currentTables[modelTable.Name]
for _, constraint := range sortConstraints(modelTable.Constraints) { for _, constraint := range sortConstraints(modelTable.Constraints) {
if constraint.Type != models.UniqueConstraint { if constraint.Type != models.UniqueConstraint {
@@ -787,7 +798,7 @@ func (w *MigrationWriter) generateUniqueConstraintScripts(model, current *models
columnExprs := make([]string, 0, len(constraint.Columns)) columnExprs := make([]string, 0, len(constraint.Columns))
for _, colName := range constraint.Columns { for _, colName := range constraint.Columns {
if col, ok := modelTable.Columns[colName]; ok { if col, ok := modelTable.Columns[colName]; ok {
columnExprs = append(columnExprs, col.SQLName()) columnExprs = append(columnExprs, quoteIdent(col.Name))
} }
} }
if len(columnExprs) == 0 { if len(columnExprs) == 0 {
@@ -837,7 +848,7 @@ func adoptableUniqueIndex(table *models.Table, constraint *models.Constraint) st
return "" return ""
} }
for i, col := range index.Columns { for i, col := range index.Columns {
if !strings.EqualFold(col, constraint.Columns[i]) { if col != constraint.Columns[i] {
return "" return ""
} }
} }
@@ -854,13 +865,13 @@ func (w *MigrationWriter) generateForeignKeyScripts(model, current *models.Schem
currentTables := make(map[string]*models.Table) currentTables := make(map[string]*models.Table)
if current != nil { if current != nil {
for _, table := range current.Tables { for _, table := range current.Tables {
currentTables[strings.ToLower(table.Name)] = table currentTables[table.Name] = table
} }
} }
// Process each model table // Process each model table
for _, modelTable := range model.Tables { for _, modelTable := range model.Tables {
currentTable := currentTables[strings.ToLower(modelTable.Name)] currentTable := currentTables[modelTable.Name]
// Process each constraint // Process each constraint
for _, constraint := range sortConstraints(modelTable.Constraints) { for _, constraint := range sortConstraints(modelTable.Constraints) {
@@ -903,10 +914,10 @@ func (w *MigrationWriter) generateForeignKeyScripts(model, current *models.Schem
SchemaName: model.Name, SchemaName: model.Name,
TableName: modelTable.Name, TableName: modelTable.Name,
ConstraintName: constraintName, ConstraintName: constraintName,
SourceColumns: strings.Join(constraint.Columns, ", "), SourceColumns: quotedIdentifiers(constraint.Columns),
TargetSchema: constraint.ReferencedSchema, TargetSchema: constraint.ReferencedSchema,
TargetTable: constraint.ReferencedTable, TargetTable: constraint.ReferencedTable,
TargetColumns: strings.Join(constraint.ReferencedColumns, ", "), TargetColumns: quotedIdentifiers(constraint.ReferencedColumns),
OnDelete: onDelete, OnDelete: onDelete,
OnUpdate: onUpdate, OnUpdate: onUpdate,
Deferral: deferral, Deferral: deferral,
@@ -938,20 +949,20 @@ func (w *MigrationWriter) generateCommentScripts(model, current *models.Schema)
currentTables := make(map[string]*models.Table) currentTables := make(map[string]*models.Table)
if current != nil { if current != nil {
for _, table := range current.Tables { for _, table := range current.Tables {
currentTables[strings.ToLower(table.Name)] = table currentTables[table.Name] = table
} }
} }
// Process each model table // Process each model table
for _, modelTable := range model.Tables { for _, modelTable := range model.Tables {
currentTable := currentTables[strings.ToLower(modelTable.Name)] currentTable := currentTables[modelTable.Name]
// Table comment (skipped when the live table already carries the same comment) // Table comment (skipped when the live table already carries the same comment)
if modelTable.Description != "" && (currentTable == nil || strings.TrimSpace(currentTable.Description) != strings.TrimSpace(modelTable.Description)) { if modelTable.Description != "" && (currentTable == nil || strings.TrimSpace(currentTable.Description) != strings.TrimSpace(modelTable.Description)) {
sql, err := w.executor.ExecuteCommentTable(CommentTableData{ sql, err := w.executor.ExecuteCommentTable(CommentTableData{
SchemaName: model.Name, SchemaName: model.Name,
TableName: modelTable.Name, TableName: modelTable.Name,
Comment: escapeQuote(modelTable.Description), Comment: modelTable.Description,
}) })
if err != nil { if err != nil {
return nil, err return nil, err
@@ -975,7 +986,7 @@ func (w *MigrationWriter) generateCommentScripts(model, current *models.Schema)
SchemaName: model.Name, SchemaName: model.Name,
TableName: modelTable.Name, TableName: modelTable.Name,
ColumnName: col.Name, ColumnName: col.Name,
Comment: escapeQuote(col.Description), Comment: col.Description,
}) })
if err != nil { if err != nil {
return nil, err return nil, err
@@ -1228,21 +1239,21 @@ func constraintsEqual(c1, c2 *models.Constraint) bool {
return false return false
} }
for i, col := range c1.Columns { for i, col := range c1.Columns {
if !strings.EqualFold(col, c2.Columns[i]) { if col != c2.Columns[i] {
return false return false
} }
} }
// For foreign keys, also compare referenced table and columns // For foreign keys, also compare referenced table and columns
if c1.Type == models.ForeignKeyConstraint { if c1.Type == models.ForeignKeyConstraint {
if !strings.EqualFold(c1.ReferencedTable, c2.ReferencedTable) { if c1.ReferencedTable != c2.ReferencedTable {
return false return false
} }
if len(c1.ReferencedColumns) != len(c2.ReferencedColumns) { if len(c1.ReferencedColumns) != len(c2.ReferencedColumns) {
return false return false
} }
for i, col := range c1.ReferencedColumns { for i, col := range c1.ReferencedColumns {
if !strings.EqualFold(col, c2.ReferencedColumns[i]) { if col != c2.ReferencedColumns[i] {
return false return false
} }
} }
@@ -1272,7 +1283,7 @@ func indexesEqual(idx1, idx2 *models.Index) bool {
return false return false
} }
for i, col := range idx1.Columns { for i, col := range idx1.Columns {
if !strings.EqualFold(col, idx2.Columns[i]) { if col != idx2.Columns[i] {
return false return false
} }
} }
@@ -1326,7 +1337,7 @@ func currentColumnHasDescription(currentTable *models.Table, col *models.Column)
return false return false
} }
for _, cc := range currentTable.Columns { for _, cc := range currentTable.Columns {
if strings.EqualFold(cc.Name, col.Name) { if cc.Name == col.Name {
return strings.TrimSpace(cc.Description) == strings.TrimSpace(col.Description) return strings.TrimSpace(cc.Description) == strings.TrimSpace(col.Description)
} }
} }
@@ -1354,7 +1365,7 @@ func primaryKeyColumnsMatch(modelTable *models.Table, current *models.Constraint
return false return false
} }
for i, col := range modelCols { for i, col := range modelCols {
if !strings.EqualFold(col, current.Columns[i]) { if col != current.Columns[i] {
return false return false
} }
} }
+32 -36
View File
@@ -89,7 +89,7 @@ func TestWriteMigration_ArrayDefault(t *testing.T) {
} }
output := buf.String() output := buf.String()
if !strings.Contains(output, "tags text[] DEFAULT '{}' NOT NULL") { if !strings.Contains(output, "\"tags\" text[] DEFAULT '{}' NOT NULL") {
t.Fatalf("expected normalized array default in migration, got:\n%s", output) t.Fatalf("expected normalized array default in migration, got:\n%s", output)
} }
if strings.Contains(output, "'''{}'''") { if strings.Contains(output, "'''{}'''") {
@@ -128,15 +128,15 @@ func TestWriteMigration_AltersColumnTypeWhenActualTypeDiffers(t *testing.T) {
} }
output := buf.String() output := buf.String()
if !strings.Contains(output, "ALTER TABLE public.learnings") || !strings.Contains(output, "ALTER COLUMN details TYPE text") { if !strings.Contains(output, "ALTER TABLE \"public\".\"learnings\"") || !strings.Contains(output, "ALTER COLUMN \"details\" TYPE text") {
t.Fatalf("expected migration to alter mismatched column type, got:\n%s", output) t.Fatalf("expected migration to alter mismatched column type, got:\n%s", output)
} }
if !strings.Contains(output, `ALTER COLUMN details TYPE text USING details::text;`) { if !strings.Contains(output, `ALTER COLUMN "details" TYPE text USING "details"::text;`) {
t.Fatalf("expected migration type alter to include USING cast, got:\n%s", output) t.Fatalf("expected migration type alter to include USING cast, got:\n%s", output)
} }
} }
func TestWriteMigration_AltersColumnTypeFallsBackToRenameAndAddOnConversionFailure(t *testing.T) { func TestWriteMigration_AltersColumnTypePreservesConversionFailure(t *testing.T) {
current := models.InitDatabase("testdb") current := models.InitDatabase("testdb")
currentSchema := models.InitSchema("public") currentSchema := models.InitSchema("public")
currentTable := models.InitTable("learnings", "public") currentTable := models.InitTable("learnings", "public")
@@ -167,17 +167,13 @@ func TestWriteMigration_AltersColumnTypeFallsBackToRenameAndAddOnConversionFailu
} }
output := buf.String() output := buf.String()
if !strings.Contains(output, "EXCEPTION WHEN OTHERS THEN") { if !strings.Contains(output, "ALTER COLUMN \"details\" TYPE integer USING \"details\"::integer;") {
t.Fatalf("expected migration to guard the type conversion with an exception handler, got:\n%s", output) t.Fatalf("expected explicit type conversion, got:\n%s", output)
} }
if !strings.Contains(output, "RENAME COLUMN details TO %I") { for _, unsafe := range []string{"EXCEPTION WHEN OTHERS", "renamed_column", "RENAME COLUMN"} {
t.Fatalf("expected migration to rename the old column (derived from the live type) on conversion failure, got:\n%s", output) if strings.Contains(output, unsafe) {
t.Fatalf("type conversion must propagate errors without replacing the column (%s):\n%s", unsafe, output)
} }
if !strings.Contains(output, "renamed_column := 'details_' || trim(both '_' from regexp_replace(lower(current_type)") {
t.Fatalf("expected migration to derive the renamed column name from the live type, got:\n%s", output)
}
if !strings.Contains(output, "ADD COLUMN details integer") {
t.Fatalf("expected migration to add a fresh column with the new type on conversion failure, got:\n%s", output)
} }
} }
@@ -214,7 +210,7 @@ func TestWriteMigration_AltersColumnNullabilityWhenNotNullDiffers(t *testing.T)
} }
output := buf.String() output := buf.String()
if !strings.Contains(output, "ALTER COLUMN rid_service_instance_type DROP NOT NULL") { if !strings.Contains(output, "ALTER COLUMN \"rid_service_instance_type\" DROP NOT NULL") {
t.Fatalf("expected migration to drop NOT NULL on existing column, got:\n%s", output) t.Fatalf("expected migration to drop NOT NULL on existing column, got:\n%s", output)
} }
} }
@@ -335,13 +331,13 @@ func TestWriteMigration_UsesStorageTypeForSerialAlterStatements(t *testing.T) {
} }
output := buf.String() output := buf.String()
if !strings.Contains(output, "ALTER COLUMN id TYPE bigint") { if !strings.Contains(output, "ALTER COLUMN \"id\" TYPE bigint") {
t.Fatalf("expected serial alter to use bigint storage type, got:\n%s", output) t.Fatalf("expected serial alter to use bigint storage type, got:\n%s", output)
} }
if strings.Contains(output, "ALTER COLUMN id TYPE bigserial;") { if strings.Contains(output, "ALTER COLUMN \"id\" TYPE bigserial;") {
t.Fatalf("did not expect invalid bigserial alter statement, got:\n%s", output) t.Fatalf("did not expect invalid bigserial alter statement, got:\n%s", output)
} }
if !strings.Contains(output, `ALTER COLUMN id TYPE bigint USING id::bigint;`) { if !strings.Contains(output, `ALTER COLUMN "id" TYPE bigint USING "id"::bigint;`) {
t.Fatalf("expected serial alter to include USING cast, got:\n%s", output) t.Fatalf("expected serial alter to include USING cast, got:\n%s", output)
} }
} }
@@ -377,7 +373,7 @@ func TestWriteMigration_ArrayAlterIncludesUsingCast(t *testing.T) {
} }
output := buf.String() output := buf.String()
if !strings.Contains(output, `ALTER COLUMN tags TYPE text[] USING tags::text[];`) { if !strings.Contains(output, `ALTER COLUMN "tags" TYPE text[] USING "tags"::text[];`) {
t.Fatalf("expected array alter to include USING cast, got:\n%s", output) t.Fatalf("expected array alter to include USING cast, got:\n%s", output)
} }
} }
@@ -414,7 +410,7 @@ func TestWriteMigration_DoesNotAlterEquivalentNormalizedColumnType(t *testing.T)
} }
output := buf.String() output := buf.String()
if strings.Contains(output, "ALTER COLUMN email TYPE") { if strings.Contains(output, "ALTER COLUMN \"email\" TYPE") {
t.Fatalf("did not expect alter type for equivalent normalized types, got:\n%s", output) t.Fatalf("did not expect alter type for equivalent normalized types, got:\n%s", output)
} }
} }
@@ -497,7 +493,7 @@ func TestWriteMigration_GinIndexOnTextUsesTrigramOperatorClass(t *testing.T) {
if !strings.Contains(output, "CREATE EXTENSION IF NOT EXISTS pg_trgm;") { if !strings.Contains(output, "CREATE EXTENSION IF NOT EXISTS pg_trgm;") {
t.Fatalf("expected trigram extension for text GIN migration index, got:\n%s", output) t.Fatalf("expected trigram extension for text GIN migration index, got:\n%s", output)
} }
if !strings.Contains(output, "USING gin (title gin_trgm_ops)") { if !strings.Contains(output, "USING gin (\"title\" gin_trgm_ops)") {
t.Fatalf("expected GIN text index to include gin_trgm_ops, got:\n%s", output) t.Fatalf("expected GIN text index to include gin_trgm_ops, got:\n%s", output)
} }
} }
@@ -518,7 +514,7 @@ func TestWriteMigration_GinIndexOnQuotedTextColumnUsesTrigramOperatorClass(t *te
index := &models.Index{ index := &models.Index{
Name: "idx_agent_personas_name_gin", Name: "idx_agent_personas_name_gin",
Type: "gin", Type: "gin",
Columns: []string{`"name"`}, Columns: []string{"name"},
} }
table.Indexes[index.Name] = index table.Indexes[index.Name] = index
@@ -537,7 +533,7 @@ func TestWriteMigration_GinIndexOnQuotedTextColumnUsesTrigramOperatorClass(t *te
} }
output := buf.String() output := buf.String()
if !strings.Contains(output, "USING gin (name gin_trgm_ops)") { if !strings.Contains(output, "USING gin (\"name\" gin_trgm_ops)") {
t.Fatalf("expected quoted text column GIN index to include gin_trgm_ops, got:\n%s", output) t.Fatalf("expected quoted text column GIN index to include gin_trgm_ops, got:\n%s", output)
} }
} }
@@ -577,7 +573,7 @@ func TestWriteMigration_GinIndexOnTextArrayDoesNotUseTrigramOperatorClass(t *tes
} }
output := buf.String() output := buf.String()
if !strings.Contains(output, "USING gin (tags array_ops)") { if !strings.Contains(output, "USING gin (\"tags\" array_ops)") {
t.Fatalf("expected GIN array index with array_ops, got:\n%s", output) t.Fatalf("expected GIN array index with array_ops, got:\n%s", output)
} }
if strings.Contains(output, "gin_trgm_ops") { if strings.Contains(output, "gin_trgm_ops") {
@@ -620,7 +616,7 @@ func TestWriteMigration_GinIndexOnJSONBUsesJSONBOperatorClass(t *testing.T) {
} }
output := buf.String() output := buf.String()
if !strings.Contains(output, "USING gin (details jsonb_ops)") { if !strings.Contains(output, "USING gin (\"details\" jsonb_ops)") {
t.Fatalf("expected GIN jsonb index to include jsonb_ops, got:\n%s", output) t.Fatalf("expected GIN jsonb index to include jsonb_ops, got:\n%s", output)
} }
if strings.Contains(output, "gin_trgm_ops") { if strings.Contains(output, "gin_trgm_ops") {
@@ -664,7 +660,7 @@ func TestWriteMigration_GinIndexOnJSONBIgnoresIncompatibleTrigramOperatorClass(t
} }
output := buf.String() output := buf.String()
if !strings.Contains(output, "USING gin (details jsonb_ops)") { if !strings.Contains(output, "USING gin (\"details\" jsonb_ops)") {
t.Fatalf("expected incompatible trigram hint on jsonb to fall back to jsonb_ops, got:\n%s", output) t.Fatalf("expected incompatible trigram hint on jsonb to fall back to jsonb_ops, got:\n%s", output)
} }
} }
@@ -725,20 +721,20 @@ func TestWriteMigration_WithAudit(t *testing.T) {
t.Logf("Generated migration with audit:\n%s", output) t.Logf("Generated migration with audit:\n%s", output)
// Verify audit tables // Verify audit tables
if !strings.Contains(output, "CREATE TABLE IF NOT EXISTS public.atevent") { if !strings.Contains(output, "CREATE TABLE IF NOT EXISTS \"public\".atevent") {
t.Error("Migration missing atevent table") t.Error("Migration missing atevent table")
} }
if !strings.Contains(output, "CREATE TABLE IF NOT EXISTS public.atdetail") { if !strings.Contains(output, "CREATE TABLE IF NOT EXISTS \"public\".atdetail") {
t.Error("Migration missing atdetail table") t.Error("Migration missing atdetail table")
} }
// Verify audit function // Verify audit function
if !strings.Contains(output, "CREATE OR REPLACE FUNCTION public.tf_audit_users()") { if !strings.Contains(output, "CREATE OR REPLACE FUNCTION \"public\".\"tf_audit_users\"()") {
t.Error("Migration missing audit function") t.Error("Migration missing audit function")
} }
// Verify audit trigger // Verify audit trigger
if !strings.Contains(output, "CREATE TRIGGER t_audit_users") { if !strings.Contains(output, "CREATE TRIGGER \"t_audit_users\"") {
t.Error("Migration missing audit trigger") t.Error("Migration missing audit trigger")
} }
@@ -770,13 +766,13 @@ func TestTemplateExecutor_CreateTable(t *testing.T) {
t.Logf("Generated SQL:\n%s", sql) t.Logf("Generated SQL:\n%s", sql)
if !strings.Contains(sql, "CREATE TABLE IF NOT EXISTS public.test_table") { if !strings.Contains(sql, "CREATE TABLE IF NOT EXISTS \"public\".\"test_table\"") {
t.Error("SQL missing CREATE TABLE statement") t.Error("SQL missing CREATE TABLE statement")
} }
if !strings.Contains(sql, "id integer NOT NULL") { if !strings.Contains(sql, "\"id\" integer NOT NULL") {
t.Error("SQL missing id column definition") t.Error("SQL missing id column definition")
} }
if !strings.Contains(sql, "name text DEFAULT 'unknown'") { if !strings.Contains(sql, "\"name\" text DEFAULT 'unknown'") {
t.Error("SQL missing name column definition") t.Error("SQL missing name column definition")
} }
} }
@@ -814,7 +810,7 @@ func TestTemplateExecutor_AuditFunction(t *testing.T) {
t.Logf("Generated SQL:\n%s", sql) t.Logf("Generated SQL:\n%s", sql)
if !strings.Contains(sql, "CREATE OR REPLACE FUNCTION public.tf_audit_users()") { if !strings.Contains(sql, "CREATE OR REPLACE FUNCTION \"public\".\"tf_audit_users\"()") {
t.Error("SQL missing function definition") t.Error("SQL missing function definition")
} }
if !strings.Contains(sql, "IF TG_OP = 'INSERT'") { if !strings.Contains(sql, "IF TG_OP = 'INSERT'") {
@@ -987,8 +983,8 @@ func TestWriteMigration_VectorAndPostGISIndexes(t *testing.T) {
"CREATE EXTENSION IF NOT EXISTS vector;", "CREATE EXTENSION IF NOT EXISTS vector;",
"vector(1536)", "vector(1536)",
"geometry(Point,4326)", "geometry(Point,4326)",
"USING ivfflat (embedding vector_cosine_ops) WITH (lists = 100)", "USING ivfflat (\"embedding\" vector_cosine_ops) WITH (lists = 100)",
"USING gist (location)", "USING gist (\"location\")",
} { } {
if !strings.Contains(output, want) { if !strings.Contains(output, want) {
t.Fatalf("expected migration to contain %q, got:\n%s", want, output) t.Fatalf("expected migration to contain %q, got:\n%s", want, output)
@@ -1185,7 +1181,7 @@ func TestWriteMigration_UniqueConstraintAdoptsSameNamedModelIndex(t *testing.T)
if got := strings.Contains(output, "UNIQUE USING INDEX"); got != tt.wantAdopt { if got := strings.Contains(output, "UNIQUE USING INDEX"); got != tt.wantAdopt {
t.Errorf("USING INDEX present = %v, want %v; output:\n%s", got, tt.wantAdopt, output) t.Errorf("USING INDEX present = %v, want %v; output:\n%s", got, tt.wantAdopt, output)
} }
if got := strings.Contains(output, "UNIQUE (guid)") || strings.Contains(output, "UNIQUE (\"guid\")"); got == tt.wantAdopt { if got := strings.Contains(output, "UNIQUE (\"guid\")"); got == tt.wantAdopt {
t.Errorf("column-list ADD CONSTRAINT present = %v, want %v; output:\n%s", got, !tt.wantAdopt, output) t.Errorf("column-list ADD CONSTRAINT present = %v, want %v; output:\n%s", got, !tt.wantAdopt, output)
} }
if got := strings.Contains(output, "DROP INDEX"); got != tt.wantDrop { if got := strings.Contains(output, "DROP INDEX"); got != tt.wantDrop {
+251
View File
@@ -0,0 +1,251 @@
package pgsql
import (
"bytes"
"context"
"os"
"path/filepath"
"strings"
"testing"
"git.warky.dev/wdevs/relspecgo/pkg/models"
"git.warky.dev/wdevs/relspecgo/pkg/writers"
)
func TestQuotedNamesAndLowercaseOption(t *testing.T) {
db := models.InitDatabase("DB\nSELECT 'header';")
s := models.InitSchema("MySchema")
tbl := models.InitTable("Select", "MySchema")
col := models.InitColumn("Key", "Select", "MySchema")
col.Type = "text"
col.Default = "Keep CASE"
tbl.Columns[col.Name] = col
s.Tables = append(s.Tables, tbl)
db.Schemas = append(db.Schemas, s)
for _, lower := range []bool{false, true} {
w := NewWriter(&writers.WriterOptions{LowercaseIdentifiers: lower})
stmts, err := w.GenerateDatabaseStatements(db)
if err != nil {
t.Fatal(err)
}
out := strings.Join(stmts, "\n")
want := `"MySchema"."Select"`
if lower {
want = `"myschema"."select"`
}
if !strings.Contains(out, want) || !strings.Contains(out, "'Keep CASE'") || strings.Contains(out, "\nSELECT 'header';") {
t.Fatalf("%s", out)
}
}
if db.Name != "DB\nSELECT 'header';" || tbl.Name != "Select" {
t.Fatal("input changed")
}
path := filepath.Join(t.TempDir(), "out.sql")
if err := os.WriteFile(path, []byte("keep"), 0o600); err != nil {
t.Fatal(err)
}
col.Name = "bad\x00name"
if err := NewWriter(&writers.WriterOptions{OutputPath: path}).WriteDatabase(db); err == nil {
t.Fatal("expected NUL rejection")
}
data, _ := os.ReadFile(path)
if string(data) != "keep" {
t.Fatal("output changed before validation")
}
}
func TestAllBlockTemplatesAvoidDelimiterCollisions(t *testing.T) {
te, err := NewTemplateExecutor(false)
if err != nil {
t.Fatal(err)
}
name := "名'\"\\\n$$ $body$ $relspec_0$ $relspec_1$"
cases := []struct {
name string
data any
}{
{"alter_column_default_with_check.tmpl", AlterColumnDefaultWithCheckData{SchemaName: name, TableName: name, ColumnName: name, SetDefault: true, DefaultValue: quote(name)}},
{"alter_column_nullability_with_check.tmpl", AlterColumnNullabilityWithCheckData{SchemaName: name, TableName: name, ColumnName: name}},
{"create_check_constraint.tmpl", CreateCheckConstraintData{SchemaName: name, TableName: name, ConstraintName: name, Expression: "true"}},
{"create_foreign_key_with_check.tmpl", CreateForeignKeyWithCheckData{SchemaName: name, TableName: name, ConstraintName: name}},
{"create_primary_key.tmpl", CreatePrimaryKeyData{SchemaName: name, TableName: name, ConstraintName: name}},
{"create_primary_key_with_autogen_check.tmpl", CreatePrimaryKeyWithAutoGenCheckData{SchemaName: name, TableName: name, ConstraintName: name}},
{"create_unique_constraint.tmpl", CreateUniqueConstraintData{SchemaName: name, TableName: name, ConstraintName: name}},
{"set_sequence_value.tmpl", SetSequenceValueData{SchemaName: name, TableName: name, SequenceName: name, ColumnName: name}},
{"audit_function.tmpl", AuditFunctionData{SchemaName: name, FunctionName: name, TableName: name, PrimaryKey: name, AuditSchema: name}},
{"audit_trigger.tmpl", AuditTriggerData{SchemaName: name, TableName: name, TriggerName: name, FunctionName: name}},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
out, err := te.execute(tc.name, tc.data)
if err != nil {
t.Fatal(err)
}
if strings.Count(out, "$relspec_2$") != 2 || !strings.Contains(out, quoteIdent(name)) || !strings.Contains(out, quote(name)) {
t.Fatalf("%s", out)
}
})
}
}
func TestLive_AdversarialNamesAndAudit(t *testing.T) {
connString := liveWriterConn(t)
base, conn := liveWriterSchema(t, connString)
ctx := context.Background()
schemaName := base + "'\"\\\n名$relspec_0$"
tableName := "select"
key := "Key'\"\\\n$relspec_0$"
valueName := "名'\"\\\n$body$"
comment := "Comment ' \\ $$ $relspec_0$\nUnicode 名"
t.Cleanup(func() { _, _ = conn.Exec(ctx, "DROP SCHEMA IF EXISTS "+quoteIdent(schemaName)+" CASCADE") })
db := models.InitDatabase("test\nSELECT invalid_header;")
s := models.InitSchema(schemaName)
tbl := models.InitTable(tableName, schemaName)
id := models.InitColumn(key, tableName, schemaName)
id.Type = "integer"
id.IsPrimaryKey = true
id.NotNull = true
tbl.Columns[key] = id
value := models.InitColumn(valueName, tableName, schemaName)
value.Type = "text"
value.Default = quote(comment)
value.Description = comment
value.Comment = comment
tbl.Columns[valueName] = value
tbl.Description = comment
tbl.Comment = comment
tbl.Constraints["unique"] = &models.Constraint{Name: "unique'\"\\\n$relspec_0$", Type: models.UniqueConstraint, Columns: []string{valueName}}
tbl.Constraints["check"] = &models.Constraint{Name: "check'\"\\\n$relspec_0$", Type: models.CheckConstraint, Expression: quoteIdent(key) + ">0"}
tbl.Indexes["index"] = &models.Index{Name: "index'\"\\\n$relspec_0$", Type: "btree", Columns: []string{valueName}}
child := models.InitTable("from", schemaName)
child.Columns[key] = &models.Column{Name: key, Type: "integer"}
child.Constraints["fk"] = &models.Constraint{Name: "foreign'\"\\\n$relspec_0$", Type: models.ForeignKeyConstraint, Columns: []string{key}, ReferencedSchema: schemaName, ReferencedTable: tableName, ReferencedColumns: []string{key}}
s.Tables = []*models.Table{tbl, child}
db.Schemas = []*models.Schema{s}
w := NewWriter(&writers.WriterOptions{})
var output bytes.Buffer
w.writer = &output
if err := w.WriteDatabase(db); err != nil {
t.Fatal(err)
}
if _, err := conn.Exec(ctx, output.String()); err != nil {
t.Fatalf("file SQL: %v\n%s", err, output.String())
}
if _, err := conn.Exec(ctx, "SET standard_conforming_strings=off"); err != nil {
t.Fatal(err)
}
statements, err := w.GenerateDatabaseStatements(db)
if err != nil {
t.Fatal(err)
}
for _, stmt := range statements {
if _, err := conn.Exec(ctx, stmt); err != nil {
t.Fatalf("statement: %v\n%s", err, stmt)
}
}
target := w.qualTable(schemaName, tableName)
if _, err := conn.Exec(ctx, "INSERT INTO "+target+" ("+quoteIdent(key)+") VALUES (1)"); err != nil {
t.Fatal(err)
}
var got string
if err := conn.QueryRow(ctx, "SELECT "+quoteIdent(valueName)+" FROM "+target).Scan(&got); err != nil || got != comment {
t.Fatalf("default: %q %v", got, err)
}
if err := conn.QueryRow(ctx, `SELECT col_description(c.oid,a.attnum) FROM pg_class c JOIN pg_namespace n ON n.oid=c.relnamespace JOIN pg_attribute a ON a.attrelid=c.oid WHERE n.nspname=$1 AND c.relname=$2 AND a.attname=$3`, schemaName, tableName, valueName).Scan(&got); err != nil || got != comment {
t.Fatalf("comment: %q %v", got, err)
}
// Migration templates must preserve the same identifier and literal boundaries.
current := models.InitDatabase("current")
currentSchema := models.InitSchema(schemaName)
currentTable := models.InitTable(tableName, schemaName)
currentValue := *value
currentValue.Type = "text"
currentTable.Columns[valueName] = &currentValue
currentID := *id
currentTable.Columns[key] = &currentID
currentTable.Constraints = tbl.Constraints
currentTable.Indexes = tbl.Indexes
currentSchema.Tables = []*models.Table{currentTable, child}
current.Schemas = []*models.Schema{currentSchema}
value.Type = "varchar"
value.Length = 200
migration, err := NewMigrationWriter(&writers.WriterOptions{})
if err != nil {
t.Fatal(err)
}
var migrationOutput bytes.Buffer
migration.writer = &migrationOutput
if err := migration.WriteMigration(db, current); err != nil {
t.Fatal(err)
}
if _, err := conn.Exec(ctx, migrationOutput.String()); err != nil {
t.Fatalf("migration: %v\n%s", err, migrationOutput.String())
}
if err := conn.QueryRow(ctx, "SELECT "+quoteIdent(valueName)+" FROM "+target+" /* after migration */").Scan(&got); err != nil || got != comment {
t.Fatalf("migration value: %q %v", got, err)
}
te := w.executor
sequenceName := "Seq'\"\\\n$relspec_0$"
sequenceSQL, err := te.ExecuteCreateSequence(CreateSequenceData{SchemaName: schemaName, SequenceName: sequenceName, Increment: 1, MinValue: 1, MaxValue: 1000, StartValue: 1, CacheSize: 1})
if err != nil {
t.Fatal(err)
}
if _, err := conn.Exec(ctx, sequenceSQL); err != nil {
t.Fatal(err)
}
setvalSQL, err := te.ExecuteSetSequenceValue(SetSequenceValueData{SchemaName: schemaName, TableName: tableName, SequenceName: sequenceName, ColumnName: key})
if err != nil {
t.Fatal(err)
}
if _, err := conn.Exec(ctx, setvalSQL); err != nil {
t.Fatalf("sequence: %v\n%s", err, setvalSQL)
}
var next int
if err := conn.QueryRow(ctx, "SELECT nextval($1::regclass)", w.qualTable(schemaName, sequenceName)).Scan(&next); err != nil || next != 2 {
t.Fatalf("sequence value: %d %v", next, err)
}
autoTable := w.qualTable(schemaName, "auto")
if _, err := conn.Exec(ctx, "CREATE TABLE "+autoTable+" (old integer PRIMARY KEY, new integer)"); err != nil {
t.Fatal(err)
}
pkSQL, err := te.ExecuteCreatePrimaryKeyWithAutoGenCheck(CreatePrimaryKeyWithAutoGenCheckData{SchemaName: schemaName, TableName: "auto", ConstraintName: "new'key", AutoGenNames: formatStringList([]string{"auto_pkey"}), Columns: quotedIdentifiers([]string{"new"}), ColumnNames: formatStringList([]string{"new"})})
if err != nil {
t.Fatal(err)
}
if _, err := conn.Exec(ctx, pkSQL); err != nil {
t.Fatalf("primary key: %v\n%s", err, pkSQL)
}
auditTables, err := te.ExecuteAuditTables(AuditTablesData{AuditSchema: schemaName})
if err != nil {
t.Fatal(err)
}
if _, err := conn.Exec(ctx, auditTables); err != nil {
t.Fatal(err)
}
config := &TableAuditConfig{AuditInsert: true, AuditUpdate: true, AuditDelete: true, TablePrefix: comment}
functionData := BuildAuditFunctionData(schemaName, tbl, id, config, schemaName, "current_user")
function, err := te.ExecuteAuditFunction(functionData)
if err != nil {
t.Fatal(err)
}
if _, err := conn.Exec(ctx, function); err != nil {
t.Fatalf("audit function: %v\n%s", err, function)
}
trigger, err := te.ExecuteAuditTrigger(AuditTriggerData{SchemaName: schemaName, TableName: tableName, FunctionName: functionData.FunctionName, TriggerName: "trigger'\"\\\n$relspec_0$", Events: "INSERT OR UPDATE OR DELETE"})
if err != nil {
t.Fatal(err)
}
if _, err := conn.Exec(ctx, trigger); err != nil {
t.Fatalf("audit trigger: %v\n%s", err, trigger)
}
for _, sql := range []string{"INSERT INTO " + target + " VALUES (2, 'second')", "UPDATE " + target + " SET " + quoteIdent(valueName) + "='third' WHERE " + quoteIdent(key) + "=2", "DELETE FROM " + target + " WHERE " + quoteIdent(key) + "=2"} {
if _, err := conn.Exec(ctx, sql); err != nil {
t.Fatal(err)
}
}
var count int
if err := conn.QueryRow(ctx, "SELECT count(*) FROM "+w.qualTable(schemaName, "atevent")+" WHERE tableprefix=$1", comment).Scan(&count); err != nil || count != 3 {
t.Fatalf("audit events: %d %v", count, err)
}
}
+1 -1
View File
@@ -110,7 +110,7 @@ func TestDiffStatements_ExistingTableNewSequenceIsSetPastData(t *testing.T) {
t.Fatalf("diffStatements failed: %v", err) t.Fatalf("diffStatements failed: %v", err)
} }
out := diffJoin(stmts) out := diffJoin(stmts)
seqIdx := strings.Index(out, "CREATE SEQUENCE IF NOT EXISTS identity.login_event_id_login_event_seq") seqIdx := strings.Index(out, "CREATE SEQUENCE IF NOT EXISTS \"identity\".\"login_event_id_login_event_seq\"")
setvalIdx := strings.Index(out, "setval(") setvalIdx := strings.Index(out, "setval(")
if seqIdx < 0 || setvalIdx < 0 || seqIdx > setvalIdx { if seqIdx < 0 || setvalIdx < 0 || seqIdx > setvalIdx {
t.Fatalf("expected sequence creation before setval, got:\n%s", out) t.Fatalf("expected sequence creation before setval, got:\n%s", out)
+13 -6
View File
@@ -2,6 +2,7 @@ package pgsql
import ( import (
"encoding/json" "encoding/json"
"errors"
"os" "os"
"path/filepath" "path/filepath"
"strings" "strings"
@@ -130,7 +131,7 @@ func TestFirstBareIdentAndHelpers(t *testing.T) {
"abc,def": "abc", "abc,def": "abc",
"abc;": "abc", "abc;": "abc",
"\n abc\tdef": "abc", "\n abc\tdef": "abc",
`"a b" c`: `"a`, `"a b" c`: `"a b"`,
" tbl (x int)": "tbl", " tbl (x int)": "tbl",
} }
for in, want := range bare { for in, want := range bare {
@@ -237,8 +238,9 @@ func TestWriteAndFinishReport(t *testing.T) {
options: &writers.WriterOptions{Metadata: map[string]interface{}{"report_path": path}}, options: &writers.WriterOptions{Metadata: map[string]interface{}{"report_path": path}},
executionReport: report, executionReport: report,
} }
if err := w.finishReport(); err != nil { var failure *ExecutionFailure
t.Fatalf("finishReport: %v", err) if err := w.finishReport(); !errors.As(err, &failure) || failure.Report != report {
t.Fatalf("expected structured execution failure, got: %v", err)
} }
data, err := os.ReadFile(path) data, err := os.ReadFile(path)
@@ -270,10 +272,15 @@ func TestWriteReportBadPath(t *testing.T) {
if err := w.writeReport(filepath.Join(t.TempDir(), "missing", "r.json")); err == nil { if err := w.writeReport(filepath.Join(t.TempDir(), "missing", "r.json")); err == nil {
t.Error("expected error for unwritable path") t.Error("expected error for unwritable path")
} }
// finishReport must swallow the report error. // Explicitly requested reports are part of successful execution.
w.options.Metadata = map[string]interface{}{"report_path": filepath.Join(t.TempDir(), "missing", "r.json")} w.options.Metadata = map[string]interface{}{"report_path": filepath.Join(t.TempDir(), "missing", "r.json")}
if err := w.finishReport(); err != nil { if err := w.finishReport(); err == nil {
t.Errorf("finishReport must not fail on report write error: %v", err) t.Error("finishReport must fail on report write error")
}
w.executionReport.FailedStatements = 1
var failure *ExecutionFailure
if err := w.finishReport(); !errors.As(err, &failure) || !strings.Contains(err.Error(), "requested execution report") {
t.Fatalf("must preserve both execution and report errors: %v", err)
} }
} }
+11 -33
View File
@@ -24,6 +24,7 @@ func TemplateFunctions() map[string]interface{} {
"escape": escape, "escape": escape,
"safe_identifier": safeIdentifier, "safe_identifier": safeIdentifier,
"quote_ident": quoteIdent, "quote_ident": quoteIdent,
"comment_text": commentText,
// Type conversion // Type conversion
"goTypeToSQL": goTypeToSQL, "goTypeToSQL": goTypeToSQL,
@@ -99,6 +100,9 @@ func indent(spaces int, text string) string {
// quote quotes a string value for SQL (escapes single quotes) // quote quotes a string value for SQL (escapes single quotes)
func quote(s string) string { func quote(s string) string {
if strings.Contains(s, "\\") {
return sqlLiteral(s)
}
return "'" + strings.ReplaceAll(s, "'", "''") + "'" return "'" + strings.ReplaceAll(s, "'", "''") + "'"
} }
@@ -125,41 +129,15 @@ func safeIdentifier(s string) string {
return strings.ToLower(safe) return strings.ToLower(safe)
} }
// quoteIdent quotes a PostgreSQL identifier if necessary // quoteIdent always quotes an identifier, preserving case and reserved words.
// Identifiers need quoting if they: func quoteIdent(s string) string { return `"` + strings.ReplaceAll(s, `"`, `""`) + `"` }
// - Start with a digit
// - Contain special characters
// - Are reserved keywords
// - Contain uppercase letters (to preserve case)
func quoteIdent(s string) string {
if s == "" {
return `""`
}
// Check if quoting is needed func quotedIdentifiers(names []string) string {
needsQuoting := unicode.IsDigit(rune(s[0])) out := make([]string, len(names))
for i, name := range names {
// Starts with digit out[i] = quoteIdent(name)
// Contains uppercase letters or special characters
for _, r := range s {
if unicode.IsUpper(r) {
needsQuoting = true
break
} }
if !unicode.IsLetter(r) && !unicode.IsDigit(r) && r != '_' { return strings.Join(out, ", ")
needsQuoting = true
break
}
}
if needsQuoting {
// Escape double quotes by doubling them
escaped := strings.ReplaceAll(s, `"`, `""`)
return `"` + escaped + `"`
}
return s
} }
// Type conversion functions // Type conversion functions
+3 -3
View File
@@ -106,7 +106,7 @@ func TestQuoteIdent(t *testing.T) {
input string input string
expected string expected string
}{ }{
{"valid_name", "valid_name"}, {"valid_name", `"valid_name"`},
{"ValidName", `"ValidName"`}, {"ValidName", `"ValidName"`},
{"123column", `"123column"`}, {"123column", `"123column"`},
{"215162_fk_constraint", `"215162_fk_constraint"`}, {"215162_fk_constraint", `"215162_fk_constraint"`},
@@ -114,8 +114,8 @@ func TestQuoteIdent(t *testing.T) {
{"user@domain", `"user@domain"`}, {"user@domain", `"user@domain"`},
{`"quoted"`, `"""quoted"""`}, {`"quoted"`, `"""quoted"""`},
{"", `""`}, {"", `""`},
{"lowercase", "lowercase"}, {"lowercase", `"lowercase"`},
{"with_underscore", "with_underscore"}, {"with_underscore", `"with_underscore"`},
} }
for _, tt := range tests { for _, tt := range tests {
+69 -155
View File
@@ -8,7 +8,6 @@ import (
"text/template" "text/template"
"git.warky.dev/wdevs/relspecgo/pkg/models" "git.warky.dev/wdevs/relspecgo/pkg/models"
"git.warky.dev/wdevs/relspecgo/pkg/writers"
) )
//go:embed templates/*.tmpl //go:embed templates/*.tmpl
@@ -29,7 +28,7 @@ func NewTemplateExecutor(flattenSchema bool) (*TemplateExecutor, error) {
} }
// qual_table returns a quoted, schema-qualified identifier. // qual_table returns a quoted, schema-qualified identifier.
// With flatten=false: "schema"."table" (or unquoted equivalents). // With flatten=false: "schema"."table".
// With flatten=true: "schema_table". // With flatten=true: "schema_table".
funcMap["qual_table"] = func(schema, name string) string { funcMap["qual_table"] = func(schema, name string) string {
if schema == "" { if schema == "" {
@@ -41,20 +40,20 @@ func NewTemplateExecutor(flattenSchema bool) (*TemplateExecutor, error) {
return quoteIdent(schema) + "." + quoteIdent(name) return quoteIdent(schema) + "." + quoteIdent(name)
} }
// qual_table_raw is the same as qual_table but without identifier quoting. funcMap["dollar_delimiter"] = func() string { return "$$" }
funcMap["qual_table_raw"] = func(schema, name string) string { tmpl := template.New("").Funcs(funcMap)
if schema == "" { entries, err := templateFS.ReadDir("templates")
return name
}
if flattenSchema {
return schema + "_" + name
}
return schema + "." + name
}
tmpl, err := template.New("").Funcs(funcMap).ParseFS(templateFS, "templates/*.tmpl")
if err != nil { if err != nil {
return nil, fmt.Errorf("failed to parse templates: %w", err) return nil, err
}
for _, entry := range entries {
source, err := templateFS.ReadFile("templates/" + entry.Name())
if err != nil {
return nil, err
}
if _, err := tmpl.New(entry.Name()).Parse(string(source)); err != nil {
return nil, fmt.Errorf("parse %s: %w", entry.Name(), err)
}
} }
return &TemplateExecutor{ return &TemplateExecutor{
@@ -62,6 +61,31 @@ func NewTemplateExecutor(flattenSchema bool) (*TemplateExecutor, error) {
}, nil }, nil
} }
// execute chooses a delimiter absent from rendered data, including SQL expressions.
// Only explicit template calls emit delimiters; user text is never rewritten.
func (te *TemplateExecutor) execute(name string, data any) (string, error) {
for n := 0; ; n++ {
delimiter := fmt.Sprintf("$relspec_%d$", n)
count := 0
tmpl, err := te.templates.Clone()
if err != nil {
return "", err
}
tmpl.Funcs(template.FuncMap{"dollar_delimiter": func() string { count++; return delimiter }})
var buf bytes.Buffer
if err := tmpl.ExecuteTemplate(&buf, name, data); err != nil {
return "", fmt.Errorf("execute %s: %w", name, err)
}
rendered := buf.String()
if strings.ContainsRune(rendered, 0) {
return "", fmt.Errorf("%s contains a NUL byte", name)
}
if strings.Count(rendered, delimiter) == count {
return rendered, nil
}
}
}
// Template data structures // Template data structures
// CreateTableData contains data for create table template // CreateTableData contains data for create table template
@@ -99,8 +123,7 @@ type AddColumnData struct {
// AlterColumnTypeWithCheckData contains data for the guarded alter column // AlterColumnTypeWithCheckData contains data for the guarded alter column
// type template, which only alters existing columns whose live type // type template, which only alters existing columns whose live type
// differs from the desired one, and falls back to renaming the old column // differs from the desired one. Conversion errors propagate unchanged.
// and adding a fresh one when the in-place conversion is not possible.
type AlterColumnTypeWithCheckData struct { type AlterColumnTypeWithCheckData struct {
SchemaName string SchemaName string
TableName string TableName string
@@ -108,6 +131,7 @@ type AlterColumnTypeWithCheckData struct {
NewType string NewType string
EquivalentTypes string EquivalentTypes string
UsingExpr string UsingExpr string
PreflightPredicate string
} }
// AlterColumnDefaultWithCheckData contains data for the guarded alter // AlterColumnDefaultWithCheckData contains data for the guarded alter
@@ -305,227 +329,117 @@ type CreatePrimaryKeyWithAutoGenCheckData struct {
// ExecuteCreateTable executes the create table template // ExecuteCreateTable executes the create table template
func (te *TemplateExecutor) ExecuteCreateTable(data CreateTableData) (string, error) { func (te *TemplateExecutor) ExecuteCreateTable(data CreateTableData) (string, error) {
var buf bytes.Buffer return te.execute("create_table.tmpl", data)
err := te.templates.ExecuteTemplate(&buf, "create_table.tmpl", data)
if err != nil {
return "", fmt.Errorf("failed to execute create_table template: %w", err)
}
return buf.String(), nil
} }
// ExecuteAddColumn executes the add column template // ExecuteAddColumn executes the add column template
func (te *TemplateExecutor) ExecuteAddColumn(data AddColumnData) (string, error) { func (te *TemplateExecutor) ExecuteAddColumn(data AddColumnData) (string, error) {
var buf bytes.Buffer return te.execute("add_column.tmpl", data)
err := te.templates.ExecuteTemplate(&buf, "add_column.tmpl", data)
if err != nil {
return "", fmt.Errorf("failed to execute add_column template: %w", err)
}
return buf.String(), nil
} }
// ExecuteAlterColumnTypeWithCheck executes the guarded alter column type // ExecuteAlterColumnTypeWithCheck executes the guarded alter column type
// template shared by the full-schema writer and the diff-based migration // template shared by the full-schema writer and the diff-based migration
// writer. // writer.
func (te *TemplateExecutor) ExecuteAlterColumnTypeWithCheck(data AlterColumnTypeWithCheckData) (string, error) { func (te *TemplateExecutor) ExecuteAlterColumnTypeWithCheck(data AlterColumnTypeWithCheckData) (string, error) {
var buf bytes.Buffer return te.execute("alter_column_type_with_check.tmpl", data)
err := te.templates.ExecuteTemplate(&buf, "alter_column_type_with_check.tmpl", data)
if err != nil {
return "", fmt.Errorf("failed to execute alter_column_type_with_check template: %w", err)
}
return buf.String(), nil
} }
// ExecuteAlterColumnDefaultWithCheck executes the guarded alter column // ExecuteAlterColumnDefaultWithCheck executes the guarded alter column
// default template shared by the full-schema writer and the diff-based // default template shared by the full-schema writer and the diff-based
// migration writer. // migration writer.
func (te *TemplateExecutor) ExecuteAlterColumnDefaultWithCheck(data AlterColumnDefaultWithCheckData) (string, error) { func (te *TemplateExecutor) ExecuteAlterColumnDefaultWithCheck(data AlterColumnDefaultWithCheckData) (string, error) {
var buf bytes.Buffer return te.execute("alter_column_default_with_check.tmpl", data)
err := te.templates.ExecuteTemplate(&buf, "alter_column_default_with_check.tmpl", data)
if err != nil {
return "", fmt.Errorf("failed to execute alter_column_default_with_check template: %w", err)
}
return buf.String(), nil
} }
// ExecuteAlterColumnNullabilityWithCheck executes the guarded alter column // ExecuteAlterColumnNullabilityWithCheck executes the guarded alter column
// nullability template. // nullability template.
func (te *TemplateExecutor) ExecuteAlterColumnNullabilityWithCheck(data AlterColumnNullabilityWithCheckData) (string, error) { func (te *TemplateExecutor) ExecuteAlterColumnNullabilityWithCheck(data AlterColumnNullabilityWithCheckData) (string, error) {
var buf bytes.Buffer return te.execute("alter_column_nullability_with_check.tmpl", data)
err := te.templates.ExecuteTemplate(&buf, "alter_column_nullability_with_check.tmpl", data)
if err != nil {
return "", fmt.Errorf("failed to execute alter_column_nullability_with_check template: %w", err)
}
return buf.String(), nil
} }
// ExecuteCreatePrimaryKey executes the create primary key template // ExecuteCreatePrimaryKey executes the create primary key template
func (te *TemplateExecutor) ExecuteCreatePrimaryKey(data CreatePrimaryKeyData) (string, error) { func (te *TemplateExecutor) ExecuteCreatePrimaryKey(data CreatePrimaryKeyData) (string, error) {
var buf bytes.Buffer return te.execute("create_primary_key.tmpl", data)
err := te.templates.ExecuteTemplate(&buf, "create_primary_key.tmpl", data)
if err != nil {
return "", fmt.Errorf("failed to execute create_primary_key template: %w", err)
}
return buf.String(), nil
} }
// ExecuteCreateIndex executes the create index template // ExecuteCreateIndex executes the create index template
func (te *TemplateExecutor) ExecuteCreateIndex(data CreateIndexData) (string, error) { func (te *TemplateExecutor) ExecuteCreateIndex(data CreateIndexData) (string, error) {
var buf bytes.Buffer return te.execute("create_index.tmpl", data)
err := te.templates.ExecuteTemplate(&buf, "create_index.tmpl", data)
if err != nil {
return "", fmt.Errorf("failed to execute create_index template: %w", err)
}
return buf.String(), nil
} }
// ExecuteCreateForeignKey executes the create foreign key template // ExecuteCreateForeignKey executes the create foreign key template
func (te *TemplateExecutor) ExecuteCreateForeignKey(data CreateForeignKeyData) (string, error) { func (te *TemplateExecutor) ExecuteCreateForeignKey(data CreateForeignKeyData) (string, error) {
var buf bytes.Buffer return te.execute("create_foreign_key.tmpl", data)
err := te.templates.ExecuteTemplate(&buf, "create_foreign_key.tmpl", data)
if err != nil {
return "", fmt.Errorf("failed to execute create_foreign_key template: %w", err)
}
return buf.String(), nil
} }
// ExecuteDropConstraint executes the drop constraint template // ExecuteDropConstraint executes the drop constraint template
func (te *TemplateExecutor) ExecuteDropConstraint(data DropConstraintData) (string, error) { func (te *TemplateExecutor) ExecuteDropConstraint(data DropConstraintData) (string, error) {
var buf bytes.Buffer return te.execute("drop_constraint.tmpl", data)
err := te.templates.ExecuteTemplate(&buf, "drop_constraint.tmpl", data)
if err != nil {
return "", fmt.Errorf("failed to execute drop_constraint template: %w", err)
}
return buf.String(), nil
} }
// ExecuteDropIndex executes the drop index template // ExecuteDropIndex executes the drop index template
func (te *TemplateExecutor) ExecuteDropIndex(data DropIndexData) (string, error) { func (te *TemplateExecutor) ExecuteDropIndex(data DropIndexData) (string, error) {
var buf bytes.Buffer return te.execute("drop_index.tmpl", data)
err := te.templates.ExecuteTemplate(&buf, "drop_index.tmpl", data)
if err != nil {
return "", fmt.Errorf("failed to execute drop_index template: %w", err)
}
return buf.String(), nil
} }
// ExecuteCommentTable executes the table comment template // ExecuteCommentTable executes the table comment template
func (te *TemplateExecutor) ExecuteCommentTable(data CommentTableData) (string, error) { func (te *TemplateExecutor) ExecuteCommentTable(data CommentTableData) (string, error) {
var buf bytes.Buffer return te.execute("comment_table.tmpl", data)
err := te.templates.ExecuteTemplate(&buf, "comment_table.tmpl", data)
if err != nil {
return "", fmt.Errorf("failed to execute comment_table template: %w", err)
}
return buf.String(), nil
} }
// ExecuteCommentColumn executes the column comment template // ExecuteCommentColumn executes the column comment template
func (te *TemplateExecutor) ExecuteCommentColumn(data CommentColumnData) (string, error) { func (te *TemplateExecutor) ExecuteCommentColumn(data CommentColumnData) (string, error) {
var buf bytes.Buffer return te.execute("comment_column.tmpl", data)
err := te.templates.ExecuteTemplate(&buf, "comment_column.tmpl", data)
if err != nil {
return "", fmt.Errorf("failed to execute comment_column template: %w", err)
}
return buf.String(), nil
} }
// ExecuteAuditTables executes the audit tables template // ExecuteAuditTables executes the audit tables template
func (te *TemplateExecutor) ExecuteAuditTables(data AuditTablesData) (string, error) { func (te *TemplateExecutor) ExecuteAuditTables(data AuditTablesData) (string, error) {
var buf bytes.Buffer return te.execute("audit_tables.tmpl", data)
err := te.templates.ExecuteTemplate(&buf, "audit_tables.tmpl", data)
if err != nil {
return "", fmt.Errorf("failed to execute audit_tables template: %w", err)
}
return buf.String(), nil
} }
// ExecuteAuditFunction executes the audit function template // ExecuteAuditFunction executes the audit function template
func (te *TemplateExecutor) ExecuteAuditFunction(data AuditFunctionData) (string, error) { func (te *TemplateExecutor) ExecuteAuditFunction(data AuditFunctionData) (string, error) {
var buf bytes.Buffer return te.execute("audit_function.tmpl", data)
err := te.templates.ExecuteTemplate(&buf, "audit_function.tmpl", data)
if err != nil {
return "", fmt.Errorf("failed to execute audit_function template: %w", err)
}
return buf.String(), nil
} }
// ExecuteAuditTrigger executes the audit trigger template // ExecuteAuditTrigger executes the audit trigger template
func (te *TemplateExecutor) ExecuteAuditTrigger(data AuditTriggerData) (string, error) { func (te *TemplateExecutor) ExecuteAuditTrigger(data AuditTriggerData) (string, error) {
var buf bytes.Buffer return te.execute("audit_trigger.tmpl", data)
err := te.templates.ExecuteTemplate(&buf, "audit_trigger.tmpl", data)
if err != nil {
return "", fmt.Errorf("failed to execute audit_trigger template: %w", err)
}
return buf.String(), nil
} }
// ExecuteCreateUniqueConstraint executes the create unique constraint template // ExecuteCreateUniqueConstraint executes the create unique constraint template
func (te *TemplateExecutor) ExecuteCreateUniqueConstraint(data CreateUniqueConstraintData) (string, error) { func (te *TemplateExecutor) ExecuteCreateUniqueConstraint(data CreateUniqueConstraintData) (string, error) {
var buf bytes.Buffer return te.execute("create_unique_constraint.tmpl", data)
err := te.templates.ExecuteTemplate(&buf, "create_unique_constraint.tmpl", data)
if err != nil {
return "", fmt.Errorf("failed to execute create_unique_constraint template: %w", err)
}
return buf.String(), nil
} }
// ExecuteCreateCheckConstraint executes the create check constraint template // ExecuteCreateCheckConstraint executes the create check constraint template
func (te *TemplateExecutor) ExecuteCreateCheckConstraint(data CreateCheckConstraintData) (string, error) { func (te *TemplateExecutor) ExecuteCreateCheckConstraint(data CreateCheckConstraintData) (string, error) {
var buf bytes.Buffer return te.execute("create_check_constraint.tmpl", data)
err := te.templates.ExecuteTemplate(&buf, "create_check_constraint.tmpl", data)
if err != nil {
return "", fmt.Errorf("failed to execute create_check_constraint template: %w", err)
}
return buf.String(), nil
} }
// ExecuteCreateForeignKeyWithCheck executes the create foreign key with check template // ExecuteCreateForeignKeyWithCheck executes the create foreign key with check template
func (te *TemplateExecutor) ExecuteCreateForeignKeyWithCheck(data CreateForeignKeyWithCheckData) (string, error) { func (te *TemplateExecutor) ExecuteCreateForeignKeyWithCheck(data CreateForeignKeyWithCheckData) (string, error) {
var buf bytes.Buffer return te.execute("create_foreign_key_with_check.tmpl", data)
err := te.templates.ExecuteTemplate(&buf, "create_foreign_key_with_check.tmpl", data)
if err != nil {
return "", fmt.Errorf("failed to execute create_foreign_key_with_check template: %w", err)
}
return buf.String(), nil
} }
// ExecuteSetSequenceValue executes the set sequence value template // ExecuteSetSequenceValue executes the set sequence value template
func (te *TemplateExecutor) ExecuteSetSequenceValue(data SetSequenceValueData) (string, error) { func (te *TemplateExecutor) ExecuteSetSequenceValue(data SetSequenceValueData) (string, error) {
var buf bytes.Buffer return te.execute("set_sequence_value.tmpl", data)
err := te.templates.ExecuteTemplate(&buf, "set_sequence_value.tmpl", data)
if err != nil {
return "", fmt.Errorf("failed to execute set_sequence_value template: %w", err)
}
return buf.String(), nil
} }
// ExecuteCreateSequence executes the create sequence template // ExecuteCreateSequence executes the create sequence template
func (te *TemplateExecutor) ExecuteCreateSequence(data CreateSequenceData) (string, error) { func (te *TemplateExecutor) ExecuteCreateSequence(data CreateSequenceData) (string, error) {
var buf bytes.Buffer return te.execute("create_sequence.tmpl", data)
err := te.templates.ExecuteTemplate(&buf, "create_sequence.tmpl", data)
if err != nil {
return "", fmt.Errorf("failed to execute create_sequence template: %w", err)
}
return buf.String(), nil
} }
// ExecuteAddColumnWithCheck executes the add column with check template // ExecuteAddColumnWithCheck executes the add column with check template
func (te *TemplateExecutor) ExecuteAddColumnWithCheck(data AddColumnWithCheckData) (string, error) { func (te *TemplateExecutor) ExecuteAddColumnWithCheck(data AddColumnWithCheckData) (string, error) {
var buf bytes.Buffer return te.execute("add_column_with_check.tmpl", data)
err := te.templates.ExecuteTemplate(&buf, "add_column_with_check.tmpl", data)
if err != nil {
return "", fmt.Errorf("failed to execute add_column_with_check template: %w", err)
}
return buf.String(), nil
} }
// ExecuteCreatePrimaryKeyWithAutoGenCheck executes the create primary key with auto-generated key check template // ExecuteCreatePrimaryKeyWithAutoGenCheck executes the create primary key with auto-generated key check template
func (te *TemplateExecutor) ExecuteCreatePrimaryKeyWithAutoGenCheck(data CreatePrimaryKeyWithAutoGenCheckData) (string, error) { func (te *TemplateExecutor) ExecuteCreatePrimaryKeyWithAutoGenCheck(data CreatePrimaryKeyWithAutoGenCheckData) (string, error) {
var buf bytes.Buffer return te.execute("create_primary_key_with_autogen_check.tmpl", data)
err := te.templates.ExecuteTemplate(&buf, "create_primary_key_with_autogen_check.tmpl", data)
if err != nil {
return "", fmt.Errorf("failed to execute create_primary_key_with_autogen_check template: %w", err)
}
return buf.String(), nil
} }
// Helper functions to build template data from models // Helper functions to build template data from models
@@ -558,7 +472,7 @@ func BuildCreateTableData(schemaName string, table *models.Table) CreateTableDat
colData.Generated, colData.Identity = columnGenerationClauses(col) colData.Generated, colData.Identity = columnGenerationClauses(col)
if col.Default != nil && colData.Generated == "" && colData.Identity == "" { if col.Default != nil && colData.Generated == "" && colData.Identity == "" {
if value, ok := col.Default.(string); ok { if value, ok := col.Default.(string); ok {
colData.Default = writers.QuoteDefaultValue(stripBackticks(value), col.Type) colData.Default = quoteDefaultValue(stripBackticks(value), col.Type)
} else { } else {
colData.Default = fmt.Sprintf("%v", col.Default) colData.Default = fmt.Sprintf("%v", col.Default)
} }
@@ -609,7 +523,7 @@ func BuildAuditFunctionData(
updateComparisons := make([]string, 0) updateComparisons := make([]string, 0)
for _, col := range auditedColumns { for _, col := range auditedColumns {
updateComparisons = append(updateComparisons, updateComparisons = append(updateComparisons,
fmt.Sprintf("old.%s IS DISTINCT FROM new.%s", col.Name, col.Name)) fmt.Sprintf("old.%s IS DISTINCT FROM new.%s", quoteIdent(col.Name), quoteIdent(col.Name)))
} }
updateCondition := strings.Join(updateComparisons, " OR ") updateCondition := strings.Join(updateComparisons, " OR ")
@@ -624,8 +538,8 @@ func BuildAuditFunctionData(
} }
} }
oldValue := fmt.Sprintf("old.%s::text", col.Name) oldValue := fmt.Sprintf("old.%s::text", quoteIdent(col.Name))
newValue := fmt.Sprintf("new.%s::text", col.Name) newValue := fmt.Sprintf("new.%s::text", quoteIdent(col.Name))
if isEncrypted { if isEncrypted {
oldValue = "'****************'" oldValue = "'****************'"
@@ -650,7 +564,7 @@ func BuildAuditFunctionData(
} }
} }
oldValue := fmt.Sprintf("old.%s::text", col.Name) oldValue := fmt.Sprintf("old.%s::text", quoteIdent(col.Name))
if isEncrypted { if isEncrypted {
oldValue = "'****************'" oldValue = "'****************'"
} }
@@ -663,7 +577,7 @@ func BuildAuditFunctionData(
tablePrefix := "NULL" tablePrefix := "NULL"
if config.TablePrefix != "" { if config.TablePrefix != "" {
tablePrefix = fmt.Sprintf("'%s'", config.TablePrefix) tablePrefix = quote(config.TablePrefix)
} }
return AuditFunctionData{ return AuditFunctionData{
@@ -1,12 +1 @@
DO $$ ALTER TABLE {{qual_table .SchemaName .TableName}} ADD COLUMN IF NOT EXISTS {{.ColumnDefinition}};
BEGIN
IF NOT EXISTS (
SELECT 1 FROM information_schema.columns
WHERE table_schema = '{{.SchemaName}}'
AND table_name = '{{.TableName}}'
AND column_name = '{{.ColumnName}}'
) THEN
ALTER TABLE {{qual_table .SchemaName .TableName}} ADD COLUMN {{.ColumnDefinition}};
END IF;
END;
$$;
@@ -1,4 +1,4 @@
DO $$ DO {{dollar_delimiter}}
DECLARE DECLARE
current_default text; current_default text;
BEGIN BEGIN
@@ -8,9 +8,9 @@ BEGIN
JOIN pg_class t ON t.oid = a.attrelid JOIN pg_class t ON t.oid = a.attrelid
JOIN pg_namespace n ON n.oid = t.relnamespace JOIN pg_namespace n ON n.oid = t.relnamespace
LEFT JOIN pg_attrdef d ON d.adrelid = a.attrelid AND d.adnum = a.attnum LEFT JOIN pg_attrdef d ON d.adrelid = a.attrelid AND d.adnum = a.attnum
WHERE n.nspname = '{{.SchemaName}}' WHERE n.nspname = {{quote .SchemaName}}
AND t.relname = '{{.TableName}}' AND t.relname = {{quote .TableName}}
AND a.attname = '{{.ColumnName}}' AND a.attname = {{quote .ColumnName}}
AND a.attnum > 0 AND a.attnum > 0
AND NOT a.attisdropped; AND NOT a.attisdropped;
@@ -26,4 +26,4 @@ BEGIN
END IF; END IF;
{{- end }} {{- end }}
END; END;
$$; {{dollar_delimiter}};
@@ -1,4 +1,4 @@
DO $$ DO {{dollar_delimiter}}
DECLARE DECLARE
current_not_null boolean; current_not_null boolean;
BEGIN BEGIN
@@ -7,9 +7,9 @@ BEGIN
FROM pg_attribute a FROM pg_attribute a
JOIN pg_class t ON t.oid = a.attrelid JOIN pg_class t ON t.oid = a.attrelid
JOIN pg_namespace n ON n.oid = t.relnamespace JOIN pg_namespace n ON n.oid = t.relnamespace
WHERE n.nspname = '{{.SchemaName}}' WHERE n.nspname = {{quote .SchemaName}}
AND t.relname = '{{.TableName}}' AND t.relname = {{quote .TableName}}
AND a.attname = '{{.ColumnName}}' AND a.attname = {{quote .ColumnName}}
AND a.attnum > 0 AND a.attnum > 0
AND NOT a.attisdropped; AND NOT a.attisdropped;
@@ -23,4 +23,4 @@ BEGIN
{{- end }} {{- end }}
END IF; END IF;
END; END;
$$; {{dollar_delimiter}};
@@ -1,30 +1,22 @@
DO $$ DO {{dollar_delimiter}}
DECLARE DECLARE
current_type text; current_type text;
renamed_column text;
BEGIN BEGIN
SELECT pg_catalog.format_type(a.atttypid, a.atttypmod) SELECT pg_catalog.format_type(a.atttypid, a.atttypmod)
INTO current_type INTO current_type
FROM pg_attribute a FROM pg_attribute a
JOIN pg_class t ON t.oid = a.attrelid JOIN pg_class t ON t.oid = a.attrelid
JOIN pg_namespace n ON n.oid = t.relnamespace JOIN pg_namespace n ON n.oid = t.relnamespace
WHERE n.nspname = '{{.SchemaName}}' WHERE n.nspname = {{quote .SchemaName}}
AND t.relname = '{{.TableName}}' AND t.relname = {{quote .TableName}}
AND a.attname = '{{.ColumnName}}' AND a.attname = {{quote .ColumnName}}
AND a.attnum > 0 AND a.attnum > 0
AND NOT a.attisdropped; AND NOT a.attisdropped;
IF current_type IS NOT NULL IF current_type IS NOT NULL
AND current_type <> ALL(ARRAY[{{.EquivalentTypes}}]) THEN AND current_type <> ALL(ARRAY[{{.EquivalentTypes}}]) THEN
BEGIN
ALTER TABLE {{qual_table .SchemaName .TableName}} ALTER TABLE {{qual_table .SchemaName .TableName}}
ALTER COLUMN {{quote_ident .ColumnName}} TYPE {{.NewType}}{{if .UsingExpr}} USING {{.UsingExpr}}{{end}}; ALTER COLUMN {{quote_ident .ColumnName}} TYPE {{.NewType}}{{if .UsingExpr}} USING {{.UsingExpr}}{{end}};
EXCEPTION WHEN OTHERS THEN
renamed_column := '{{.ColumnName}}_' || trim(both '_' from regexp_replace(lower(current_type), '[^a-z0-9]+', '_', 'g'));
EXECUTE format('ALTER TABLE {{qual_table .SchemaName .TableName}} RENAME COLUMN {{quote_ident .ColumnName}} TO %I', renamed_column);
ALTER TABLE {{qual_table .SchemaName .TableName}}
ADD COLUMN {{quote_ident .ColumnName}} {{.NewType}};
END;
END IF; END IF;
END; END;
$$; {{dollar_delimiter}};
+16 -16
View File
@@ -1,8 +1,8 @@
CREATE OR REPLACE FUNCTION {{qual_table_raw .SchemaName .FunctionName}}() CREATE OR REPLACE FUNCTION {{qual_table .SchemaName .FunctionName}}()
RETURNS trigger AS RETURNS trigger AS
$body$ {{dollar_delimiter}}
DECLARE DECLARE
m_funcname text = '{{.FunctionName}}'; m_funcname text = {{quote .FunctionName}};
m_user text; m_user text;
m_atevent integer; m_atevent integer;
BEGIN BEGIN
@@ -23,8 +23,8 @@ BEGIN
{{- if .AuditInsert}} {{- if .AuditInsert}}
IF TG_OP = 'INSERT' THEN IF TG_OP = 'INSERT' THEN
-- Record INSERT -- Record INSERT
INSERT INTO {{.AuditSchema}}.atevent (tablename, tableprefix, rid_parent, changeuser, changedate, changetime, actionx) INSERT INTO {{quote_ident .AuditSchema}}.atevent (tablename, tableprefix, rid_parent, changeuser, changedate, changetime, actionx)
VALUES ('{{.TableName}}', {{.TablePrefix}}, new.{{.PrimaryKey}}, m_user, CURRENT_DATE, CURRENT_TIME, 1) VALUES ({{quote .TableName}}, {{.TablePrefix}}, new.{{quote_ident .PrimaryKey}}, m_user, CURRENT_DATE, CURRENT_TIME, 1)
RETURNING rid_atevent INTO m_atevent; RETURNING rid_atevent INTO m_atevent;
{{- end}} {{- end}}
@@ -32,15 +32,15 @@ BEGIN
ELSIF TG_OP = 'UPDATE' THEN ELSIF TG_OP = 'UPDATE' THEN
-- Check if any audited columns changed -- Check if any audited columns changed
IF ({{.UpdateCondition}}) THEN IF ({{.UpdateCondition}}) THEN
INSERT INTO {{.AuditSchema}}.atevent (tablename, tableprefix, rid_parent, changeuser, changedate, changetime, actionx) INSERT INTO {{quote_ident .AuditSchema}}.atevent (tablename, tableprefix, rid_parent, changeuser, changedate, changetime, actionx)
VALUES ('{{.TableName}}', {{.TablePrefix}}, new.{{.PrimaryKey}}, m_user, CURRENT_DATE, CURRENT_TIME, 2) VALUES ({{quote .TableName}}, {{.TablePrefix}}, new.{{quote_ident .PrimaryKey}}, m_user, CURRENT_DATE, CURRENT_TIME, 2)
RETURNING rid_atevent INTO m_atevent; RETURNING rid_atevent INTO m_atevent;
-- Record column changes -- Record column changes
{{- range .UpdateColumns}} {{- range .UpdateColumns}}
IF (old.{{.Name}} IS DISTINCT FROM new.{{.Name}}) THEN IF (old.{{quote_ident .Name}} IS DISTINCT FROM new.{{quote_ident .Name}}) THEN
INSERT INTO {{$.AuditSchema}}.atdetail(rid_atevent, datacolumn, changedfrom, changedto) INSERT INTO {{quote_ident $.AuditSchema}}.atdetail(rid_atevent, datacolumn, changedfrom, changedto)
VALUES (m_atevent, '{{.Name}}', substr({{.OldValue}}, 1, 1000), substr({{.NewValue}}, 1, 1000)); VALUES (m_atevent, {{quote .Name}}, substr({{.OldValue}}, 1, 1000), substr({{.NewValue}}, 1, 1000));
END IF; END IF;
{{- end}} {{- end}}
END IF; END IF;
@@ -49,14 +49,14 @@ BEGIN
{{- if .AuditDelete}} {{- if .AuditDelete}}
ELSIF TG_OP = 'DELETE' THEN ELSIF TG_OP = 'DELETE' THEN
-- Record DELETE -- Record DELETE
INSERT INTO {{.AuditSchema}}.atevent (tablename, tableprefix, rid_parent, rid_deletedparent, changeuser, changedate, changetime, actionx) INSERT INTO {{quote_ident .AuditSchema}}.atevent (tablename, tableprefix, rid_parent, rid_deletedparent, changeuser, changedate, changetime, actionx)
VALUES ('{{.TableName}}', {{.TablePrefix}}, old.{{.PrimaryKey}}, old.{{.PrimaryKey}}, m_user, CURRENT_DATE, CURRENT_TIME, 3) VALUES ({{quote .TableName}}, {{.TablePrefix}}, old.{{quote_ident .PrimaryKey}}, old.{{quote_ident .PrimaryKey}}, m_user, CURRENT_DATE, CURRENT_TIME, 3)
RETURNING rid_atevent INTO m_atevent; RETURNING rid_atevent INTO m_atevent;
-- Record deleted column values -- Record deleted column values
{{- range .DeleteColumns}} {{- range .DeleteColumns}}
INSERT INTO {{$.AuditSchema}}.atdetail(rid_atevent, datacolumn, changedfrom, changedto) INSERT INTO {{quote_ident $.AuditSchema}}.atdetail(rid_atevent, datacolumn, changedfrom, changedto)
VALUES (m_atevent, '{{.Name}}', substr({{.OldValue}}, 1, 1000), NULL); VALUES (m_atevent, {{quote .Name}}, substr({{.OldValue}}, 1, 1000), NULL);
{{- end}} {{- end}}
{{- end}} {{- end}}
END IF; END IF;
@@ -76,9 +76,9 @@ EXCEPTION
RAISE WARNING 'Audit function % failed: %', m_funcname, SQLERRM; RAISE WARNING 'Audit function % failed: %', m_funcname, SQLERRM;
RETURN NULL; RETURN NULL;
END; END;
$body$ {{dollar_delimiter}}
LANGUAGE plpgsql LANGUAGE plpgsql
VOLATILE VOLATILE
SECURITY DEFINER; SECURITY DEFINER;
COMMENT ON FUNCTION {{qual_table_raw .SchemaName .FunctionName}}() IS 'Audit trigger function for table {{qual_table_raw .SchemaName .TableName}}'; COMMENT ON FUNCTION {{qual_table .SchemaName .FunctionName}}() IS {{quote (printf "Audit trigger function for table %s" (qual_table .SchemaName .TableName))}};
+24 -24
View File
@@ -1,5 +1,5 @@
-- Audit Event Header Table -- Audit Event Header Table
CREATE TABLE IF NOT EXISTS {{.AuditSchema}}.atevent ( CREATE TABLE IF NOT EXISTS {{quote_ident .AuditSchema}}.atevent (
rid_atevent serial PRIMARY KEY, rid_atevent serial PRIMARY KEY,
tablename text NOT NULL, tablename text NOT NULL,
tableprefix text, tableprefix text,
@@ -12,38 +12,38 @@ CREATE TABLE IF NOT EXISTS {{.AuditSchema}}.atevent (
CONSTRAINT ck_atevent_action CHECK (actionx IN (1, 2, 3)) CONSTRAINT ck_atevent_action CHECK (actionx IN (1, 2, 3))
); );
CREATE INDEX IF NOT EXISTS idx_atevent_tablename ON {{.AuditSchema}}.atevent(tablename); CREATE INDEX IF NOT EXISTS idx_atevent_tablename ON {{quote_ident .AuditSchema}}.atevent(tablename);
CREATE INDEX IF NOT EXISTS idx_atevent_rid_parent ON {{.AuditSchema}}.atevent(rid_parent); CREATE INDEX IF NOT EXISTS idx_atevent_rid_parent ON {{quote_ident .AuditSchema}}.atevent(rid_parent);
CREATE INDEX IF NOT EXISTS idx_atevent_changedate ON {{.AuditSchema}}.atevent(changedate); CREATE INDEX IF NOT EXISTS idx_atevent_changedate ON {{quote_ident .AuditSchema}}.atevent(changedate);
CREATE INDEX IF NOT EXISTS idx_atevent_changeuser ON {{.AuditSchema}}.atevent(changeuser); CREATE INDEX IF NOT EXISTS idx_atevent_changeuser ON {{quote_ident .AuditSchema}}.atevent(changeuser);
COMMENT ON TABLE {{.AuditSchema}}.atevent IS 'Audit trail header table - tracks all data changes'; COMMENT ON TABLE {{quote_ident .AuditSchema}}.atevent IS 'Audit trail header table - tracks all data changes';
COMMENT ON COLUMN {{.AuditSchema}}.atevent.rid_atevent IS 'Audit event ID'; COMMENT ON COLUMN {{quote_ident .AuditSchema}}.atevent.rid_atevent IS 'Audit event ID';
COMMENT ON COLUMN {{.AuditSchema}}.atevent.tablename IS 'Name of the table that was modified'; COMMENT ON COLUMN {{quote_ident .AuditSchema}}.atevent.tablename IS 'Name of the table that was modified';
COMMENT ON COLUMN {{.AuditSchema}}.atevent.rid_parent IS 'Primary key value of the modified record'; COMMENT ON COLUMN {{quote_ident .AuditSchema}}.atevent.rid_parent IS 'Primary key value of the modified record';
COMMENT ON COLUMN {{.AuditSchema}}.atevent.rid_deletedparent IS 'Parent reference for deleted records'; COMMENT ON COLUMN {{quote_ident .AuditSchema}}.atevent.rid_deletedparent IS 'Parent reference for deleted records';
COMMENT ON COLUMN {{.AuditSchema}}.atevent.changeuser IS 'User who made the change'; COMMENT ON COLUMN {{quote_ident .AuditSchema}}.atevent.changeuser IS 'User who made the change';
COMMENT ON COLUMN {{.AuditSchema}}.atevent.changedate IS 'Date of change'; COMMENT ON COLUMN {{quote_ident .AuditSchema}}.atevent.changedate IS 'Date of change';
COMMENT ON COLUMN {{.AuditSchema}}.atevent.changetime IS 'Time of change'; COMMENT ON COLUMN {{quote_ident .AuditSchema}}.atevent.changetime IS 'Time of change';
COMMENT ON COLUMN {{.AuditSchema}}.atevent.actionx IS 'Action type: 1=INSERT, 2=UPDATE, 3=DELETE'; COMMENT ON COLUMN {{quote_ident .AuditSchema}}.atevent.actionx IS 'Action type: 1=INSERT, 2=UPDATE, 3=DELETE';
-- Audit Event Detail Table -- Audit Event Detail Table
CREATE TABLE IF NOT EXISTS {{.AuditSchema}}.atdetail ( CREATE TABLE IF NOT EXISTS {{quote_ident .AuditSchema}}.atdetail (
rid_atdetail serial PRIMARY KEY, rid_atdetail serial PRIMARY KEY,
rid_atevent integer NOT NULL, rid_atevent integer NOT NULL,
datacolumn text NOT NULL, datacolumn text NOT NULL,
changedfrom text, changedfrom text,
changedto text, changedto text,
CONSTRAINT fk_atdetail_atevent FOREIGN KEY (rid_atevent) CONSTRAINT fk_atdetail_atevent FOREIGN KEY (rid_atevent)
REFERENCES {{.AuditSchema}}.atevent(rid_atevent) ON DELETE CASCADE REFERENCES {{quote_ident .AuditSchema}}.atevent(rid_atevent) ON DELETE CASCADE
); );
CREATE INDEX IF NOT EXISTS idx_atdetail_rid_atevent ON {{.AuditSchema}}.atdetail(rid_atevent); CREATE INDEX IF NOT EXISTS idx_atdetail_rid_atevent ON {{quote_ident .AuditSchema}}.atdetail(rid_atevent);
CREATE INDEX IF NOT EXISTS idx_atdetail_datacolumn ON {{.AuditSchema}}.atdetail(datacolumn); CREATE INDEX IF NOT EXISTS idx_atdetail_datacolumn ON {{quote_ident .AuditSchema}}.atdetail(datacolumn);
COMMENT ON TABLE {{.AuditSchema}}.atdetail IS 'Audit trail detail table - stores individual column changes'; COMMENT ON TABLE {{quote_ident .AuditSchema}}.atdetail IS 'Audit trail detail table - stores individual column changes';
COMMENT ON COLUMN {{.AuditSchema}}.atdetail.rid_atdetail IS 'Audit detail ID'; COMMENT ON COLUMN {{quote_ident .AuditSchema}}.atdetail.rid_atdetail IS 'Audit detail ID';
COMMENT ON COLUMN {{.AuditSchema}}.atdetail.rid_atevent IS 'Reference to audit event'; COMMENT ON COLUMN {{quote_ident .AuditSchema}}.atdetail.rid_atevent IS 'Reference to audit event';
COMMENT ON COLUMN {{.AuditSchema}}.atdetail.datacolumn IS 'Name of the column that changed'; COMMENT ON COLUMN {{quote_ident .AuditSchema}}.atdetail.datacolumn IS 'Name of the column that changed';
COMMENT ON COLUMN {{.AuditSchema}}.atdetail.changedfrom IS 'Old value before change'; COMMENT ON COLUMN {{quote_ident .AuditSchema}}.atdetail.changedfrom IS 'Old value before change';
COMMENT ON COLUMN {{.AuditSchema}}.atdetail.changedto IS 'New value after change'; COMMENT ON COLUMN {{quote_ident .AuditSchema}}.atdetail.changedto IS 'New value after change';
@@ -1,16 +1,16 @@
DO $$ DO {{dollar_delimiter}}
BEGIN BEGIN
IF NOT EXISTS ( IF NOT EXISTS (
SELECT 1 SELECT 1
FROM pg_trigger FROM pg_trigger
WHERE tgname = '{{.TriggerName}}' WHERE tgname = {{quote .TriggerName}}
AND tgrelid = '{{qual_table_raw .SchemaName .TableName}}'::regclass AND tgrelid = {{quote (qual_table .SchemaName .TableName)}}::regclass
) THEN ) THEN
CREATE TRIGGER {{.TriggerName}} CREATE TRIGGER {{quote_ident .TriggerName}}
AFTER {{.Events}} AFTER {{.Events}}
ON {{qual_table_raw .SchemaName .TableName}} ON {{qual_table .SchemaName .TableName}}
FOR EACH ROW FOR EACH ROW
EXECUTE FUNCTION {{qual_table_raw .SchemaName .FunctionName}}(); EXECUTE FUNCTION {{qual_table .SchemaName .FunctionName}}();
END IF; END IF;
END; END;
$$; {{dollar_delimiter}};
@@ -1,39 +1,39 @@
{{/* Base constraint template */}} {{/* Base constraint template */}}
{{- define "constraint_base" -}} {{- define "constraint_base" -}}
ALTER TABLE {{qual_table_raw .SchemaName .TableName}} ALTER TABLE {{qual_table .SchemaName .TableName}}
ADD CONSTRAINT {{.ConstraintName}} ADD CONSTRAINT {{quote_ident .ConstraintName}}
{{block "constraint_definition" .}}{{end}}; {{block "constraint_definition" .}}{{end}};
{{- end -}} {{- end -}}
{{/* Drop constraint with check */}} {{/* Drop constraint with check */}}
{{- define "drop_constraint_safe" -}} {{- define "drop_constraint_safe" -}}
DO $$ DO {{dollar_delimiter}}
BEGIN BEGIN
IF EXISTS ( IF EXISTS (
SELECT 1 FROM information_schema.table_constraints SELECT 1 FROM information_schema.table_constraints
WHERE table_schema = '{{.SchemaName}}' WHERE table_schema = {{quote .SchemaName}}
AND table_name = '{{.TableName}}' AND table_name = {{quote .TableName}}
AND constraint_name = '{{.ConstraintName}}' AND constraint_name = {{quote .ConstraintName}}
) THEN ) THEN
ALTER TABLE {{qual_table_raw .SchemaName .TableName}} ALTER TABLE {{qual_table .SchemaName .TableName}}
DROP CONSTRAINT {{.ConstraintName}}; DROP CONSTRAINT {{quote_ident .ConstraintName}};
END IF; END IF;
END; END;
$$; {{dollar_delimiter}};
{{- end -}} {{- end -}}
{{/* Add constraint with existence check */}} {{/* Add constraint with existence check */}}
{{- define "add_constraint_safe" -}} {{- define "add_constraint_safe" -}}
DO $$ DO {{dollar_delimiter}}
BEGIN BEGIN
IF NOT EXISTS ( IF NOT EXISTS (
SELECT 1 FROM information_schema.table_constraints SELECT 1 FROM information_schema.table_constraints
WHERE table_schema = '{{.SchemaName}}' WHERE table_schema = {{quote .SchemaName}}
AND table_name = '{{.TableName}}' AND table_name = {{quote .TableName}}
AND constraint_name = '{{.ConstraintName}}' AND constraint_name = {{quote .ConstraintName}}
) THEN ) THEN
{{template "constraint_base" .}} {{template "constraint_base" .}}
END IF; END IF;
END; END;
$$; {{dollar_delimiter}};
{{- end -}} {{- end -}}
+8 -8
View File
@@ -1,23 +1,23 @@
{{/* Base DDL template with common structure */}} {{/* Base DDL template with common structure */}}
{{- define "ddl_header" -}} {{- define "ddl_header" -}}
-- DDL Operation: {{.Operation}} -- DDL Operation: {{comment_text .Operation}}
-- Schema: {{.Schema}} -- Schema: {{comment_text .Schema}}
-- Object: {{.ObjectName}} -- Object: {{comment_text .ObjectName}}
{{- end -}} {{- end -}}
{{- define "ddl_footer" -}} {{- define "ddl_footer" -}}
-- End of {{.Operation}} -- End of {{comment_text .Operation}}
{{- end -}} {{- end -}}
{{/* Base ALTER TABLE structure */}} {{/* Base ALTER TABLE structure */}}
{{- define "alter_table_base" -}} {{- define "alter_table_base" -}}
ALTER TABLE {{qual_table_raw .SchemaName .TableName}} ALTER TABLE {{qual_table .SchemaName .TableName}}
{{block "alter_operation" .}}{{end}}; {{block "alter_operation" .}}{{end}};
{{- end -}} {{- end -}}
{{/* Common existence check pattern */}} {{/* Common existence check pattern */}}
{{- define "exists_check" -}} {{- define "exists_check" -}}
DO $$ DO {{dollar_delimiter}}
BEGIN BEGIN
IF NOT EXISTS ( IF NOT EXISTS (
{{block "exists_query" .}}{{end}} {{block "exists_query" .}}{{end}}
@@ -25,10 +25,10 @@ BEGIN
{{block "create_statement" .}}{{end}} {{block "create_statement" .}}{{end}}
END IF; END IF;
END; END;
$$; {{dollar_delimiter}};
{{- end -}} {{- end -}}
{{/* Common drop pattern */}} {{/* Common drop pattern */}}
{{- define "drop_if_exists" -}} {{- define "drop_if_exists" -}}
{{block "drop_type" .}}{{end}} IF EXISTS {{qual_table_raw .SchemaName .ObjectName}}; {{block "drop_type" .}}{{end}} IF EXISTS {{qual_table .SchemaName .ObjectName}};
{{- end -}} {{- end -}}
@@ -1 +1 @@
COMMENT ON COLUMN {{qual_table .SchemaName .TableName}}.{{quote_ident .ColumnName}} IS '{{.Comment}}'; COMMENT ON COLUMN {{qual_table .SchemaName .TableName}}.{{quote_ident .ColumnName}} IS {{quote .Comment}};
@@ -1 +1 @@
COMMENT ON TABLE {{qual_table .SchemaName .TableName}} IS '{{.Comment}}'; COMMENT ON TABLE {{qual_table .SchemaName .TableName}} IS {{quote .Comment}};
@@ -1,12 +1,12 @@
DO $$ DO {{dollar_delimiter}}
BEGIN BEGIN
IF NOT EXISTS ( IF NOT EXISTS (
SELECT 1 FROM information_schema.table_constraints SELECT 1 FROM information_schema.table_constraints
WHERE table_schema = '{{.SchemaName}}' WHERE table_schema = {{quote .SchemaName}}
AND table_name = '{{.TableName}}' AND table_name = {{quote .TableName}}
AND constraint_name = '{{.ConstraintName}}' AND constraint_name = {{quote .ConstraintName}}
) THEN ) THEN
ALTER TABLE {{qual_table .SchemaName .TableName}} ADD CONSTRAINT {{quote_ident .ConstraintName}} CHECK ({{.Expression}}); ALTER TABLE {{qual_table .SchemaName .TableName}} ADD CONSTRAINT {{quote_ident .ConstraintName}} CHECK ({{.Expression}});
END IF; END IF;
END; END;
$$; {{dollar_delimiter}};
@@ -1,10 +1,10 @@
DO $$ DO {{dollar_delimiter}}
BEGIN BEGIN
IF NOT EXISTS ( IF NOT EXISTS (
SELECT 1 FROM information_schema.table_constraints SELECT 1 FROM information_schema.table_constraints
WHERE table_schema = '{{.SchemaName}}' WHERE table_schema = {{quote .SchemaName}}
AND table_name = '{{.TableName}}' AND table_name = {{quote .TableName}}
AND constraint_name = '{{.ConstraintName}}' AND constraint_name = {{quote .ConstraintName}}
) THEN ) THEN
ALTER TABLE {{qual_table .SchemaName .TableName}} ALTER TABLE {{qual_table .SchemaName .TableName}}
ADD CONSTRAINT {{quote_ident .ConstraintName}} ADD CONSTRAINT {{quote_ident .ConstraintName}}
@@ -15,4 +15,4 @@ BEGIN
{{.Deferral}}{{end}}; {{.Deferral}}{{end}};
END IF; END IF;
END; END;
$$; {{dollar_delimiter}};
@@ -1,13 +1,13 @@
DO $$ DO {{dollar_delimiter}}
BEGIN BEGIN
IF NOT EXISTS ( IF NOT EXISTS (
SELECT 1 FROM information_schema.table_constraints SELECT 1 FROM information_schema.table_constraints
WHERE table_schema = '{{.SchemaName}}' WHERE table_schema = {{quote .SchemaName}}
AND table_name = '{{.TableName}}' AND table_name = {{quote .TableName}}
AND constraint_name = '{{.ConstraintName}}' AND constraint_name = {{quote .ConstraintName}}
) THEN ) THEN
ALTER TABLE {{qual_table .SchemaName .TableName}} ALTER TABLE {{qual_table .SchemaName .TableName}}
ADD CONSTRAINT {{quote_ident .ConstraintName}} PRIMARY KEY ({{.Columns}}); ADD CONSTRAINT {{quote_ident .ConstraintName}} PRIMARY KEY ({{.Columns}});
END IF; END IF;
END; END;
$$; {{dollar_delimiter}};
@@ -1,4 +1,4 @@
DO $$ DO {{dollar_delimiter}}
DECLARE DECLARE
current_pk_name text; current_pk_name text;
current_pk_matches boolean := false; current_pk_matches boolean := false;
@@ -16,22 +16,22 @@ BEGIN
ON a.attrelid = t.oid ON a.attrelid = t.oid
AND a.attnum = cols.attnum AND a.attnum = cols.attnum
WHERE c.contype = 'p' WHERE c.contype = 'p'
AND n.nspname = '{{.SchemaName}}' AND n.nspname = {{quote .SchemaName}}
AND t.relname = '{{.TableName}}' AND t.relname = {{quote .TableName}}
ORDER BY cols.ord ORDER BY cols.ord
), ),
ARRAY[]::text[] ARRAY[]::text[]
) = ARRAY[{{.ColumnNames}}] ) = ARRAY[{{.ColumnNames}}]
INTO current_pk_name, current_pk_matches INTO current_pk_name, current_pk_matches
FROM information_schema.table_constraints tc FROM information_schema.table_constraints tc
WHERE tc.table_schema = '{{.SchemaName}}' WHERE tc.table_schema = {{quote .SchemaName}}
AND tc.table_name = '{{.TableName}}' AND tc.table_name = {{quote .TableName}}
AND tc.constraint_type = 'PRIMARY KEY'; AND tc.constraint_type = 'PRIMARY KEY';
IF current_pk_name IS NOT NULL IF current_pk_name IS NOT NULL
AND NOT current_pk_matches AND NOT current_pk_matches
AND current_pk_name IN ({{.AutoGenNames}}) THEN AND current_pk_name IN ({{.AutoGenNames}}) THEN
EXECUTE 'ALTER TABLE {{qual_table .SchemaName .TableName}} DROP CONSTRAINT ' || quote_ident(current_pk_name) || ' CASCADE'; EXECUTE {{quote (printf "ALTER TABLE %s DROP CONSTRAINT " (qual_table .SchemaName .TableName))}} || quote_ident(current_pk_name) || ' CASCADE';
END IF; END IF;
-- Add the desired primary key only when no matching primary key already exists. -- Add the desired primary key only when no matching primary key already exists.
@@ -40,4 +40,4 @@ BEGIN
ALTER TABLE {{qual_table .SchemaName .TableName}} ADD CONSTRAINT {{quote_ident .ConstraintName}} PRIMARY KEY ({{.Columns}}); ALTER TABLE {{qual_table .SchemaName .TableName}} ADD CONSTRAINT {{quote_ident .ConstraintName}} PRIMARY KEY ({{.Columns}});
END IF; END IF;
END; END;
$$; {{dollar_delimiter}};
@@ -1,12 +1,12 @@
DO $$ DO {{dollar_delimiter}}
BEGIN BEGIN
IF NOT EXISTS ( IF NOT EXISTS (
SELECT 1 FROM information_schema.table_constraints SELECT 1 FROM information_schema.table_constraints
WHERE table_schema = '{{.SchemaName}}' WHERE table_schema = {{quote .SchemaName}}
AND table_name = '{{.TableName}}' AND table_name = {{quote .TableName}}
AND constraint_name = '{{.ConstraintName}}' AND constraint_name = {{quote .ConstraintName}}
) THEN ) THEN
ALTER TABLE {{qual_table .SchemaName .TableName}} ADD CONSTRAINT {{quote_ident .ConstraintName}} UNIQUE {{if .UsingIndex}}USING INDEX {{quote_ident .UsingIndex}}{{else}}({{.Columns}}){{end}}; ALTER TABLE {{qual_table .SchemaName .TableName}} ADD CONSTRAINT {{quote_ident .ConstraintName}} UNIQUE {{if .UsingIndex}}USING INDEX {{quote_ident .UsingIndex}}{{else}}({{.Columns}}){{end}};
END IF; END IF;
END; END;
$$; {{dollar_delimiter}};
+2 -2
View File
@@ -2,7 +2,7 @@
{{/* Column definition fragment */}} {{/* Column definition fragment */}}
{{- define "column_definition" -}} {{- define "column_definition" -}}
{{.Name}} {{.Type}} {{quote_ident .Name}} {{.Type}}
{{- if .Generated}} GENERATED ALWAYS AS ({{.Generated}}) STORED{{end}} {{- if .Generated}} GENERATED ALWAYS AS ({{.Generated}}) STORED{{end}}
{{- if .Identity}} GENERATED {{.Identity}} AS IDENTITY{{end}} {{- if .Identity}} GENERATED {{.Identity}} AS IDENTITY{{end}}
{{- if and .Default (not .Generated) (not .Identity)}} DEFAULT {{.Default}}{{end}} {{- if and .Default (not .Generated) (not .Identity)}} DEFAULT {{.Default}}{{end}}
@@ -18,7 +18,7 @@
{{/* Qualified table name */}} {{/* Qualified table name */}}
{{- define "qualified_table" -}} {{- define "qualified_table" -}}
{{qual_table_raw .SchemaName .TableName}} {{qual_table .SchemaName .TableName}}
{{- end -}} {{- end -}}
{{/* Index method clause */}} {{/* Index method clause */}}
@@ -1,4 +1,4 @@
DO $$ DO {{dollar_delimiter}}
DECLARE DECLARE
m_cnt bigint; m_cnt bigint;
m_next bigint; m_next bigint;
@@ -6,8 +6,8 @@ BEGIN
IF EXISTS ( IF EXISTS (
SELECT 1 FROM pg_class c SELECT 1 FROM pg_class c
INNER JOIN pg_namespace n ON n.oid = c.relnamespace INNER JOIN pg_namespace n ON n.oid = c.relnamespace
WHERE c.relname = '{{.SequenceName}}' WHERE c.relname = {{quote .SequenceName}}
AND n.nspname = '{{.SchemaName}}' AND n.nspname = {{quote .SchemaName}}
AND c.relkind = 'S' AND c.relkind = 'S'
) THEN ) THEN
SELECT COALESCE(MAX({{quote_ident .ColumnName}}), 0) + 1 SELECT COALESCE(MAX({{quote_ident .ColumnName}}), 0) + 1
@@ -20,8 +20,8 @@ BEGIN
-- Only move the sequence forward; never rewind one that is already past the data. -- Only move the sequence forward; never rewind one that is already past the data.
IF m_cnt > m_next THEN IF m_cnt > m_next THEN
PERFORM setval('{{qual_table_raw .SchemaName .SequenceName}}'::regclass, m_cnt, false); PERFORM setval({{quote (qual_table .SchemaName .SequenceName)}}::regclass, m_cnt, false);
END IF; END IF;
END IF; END IF;
END; END;
$$; {{dollar_delimiter}};
+91
View File
@@ -0,0 +1,91 @@
package pgsql
import (
"fmt"
"strings"
"github.com/jackc/pgx/v5"
"git.warky.dev/wdevs/relspecgo/pkg/models"
)
// conversionData accepts executable SQL only through explicitly named column metadata.
func conversionData(schema, table string, col *models.Column) (AlterColumnTypeWithCheckData, error) {
target := effectiveAlterColumnSQLType(col)
data := AlterColumnTypeWithCheckData{
SchemaName: schema, TableName: table, ColumnName: col.Name,
NewType: target, EquivalentTypes: equivalentTypeListSQL(target),
UsingExpr: buildAlterColumnUsingExpression(col.Name, target),
}
for key, destination := range map[string]*string{"pgsql_using": &data.UsingExpr, "pgsql_preflight": &data.PreflightPredicate} {
if value, exists := col.Metadata[key]; exists {
expression, ok := value.(string)
if !ok || strings.TrimSpace(expression) == "" {
return data, fmt.Errorf("%s.%s.%s metadata %s must be a nonempty SQL expression", schema, table, col.Name, key)
}
*destination = expression
}
}
return data, nil
}
// sqlLiteral is independent of the connection's standard_conforming_strings setting.
func sqlLiteral(value string) string {
return "E'" + strings.NewReplacer("\\", "\\\\", "'", "''").Replace(value) + "'"
}
func dollarBlock(body, prefix string) string {
for i := 0; ; i++ {
delimiter := fmt.Sprintf("$%s_%d$", prefix, i)
if !strings.Contains(body, delimiter) {
return "DO " + delimiter + "\n" + body + "\n" + delimiter + ";"
}
}
}
// preflightColumnType checks every value before any schema changes. It is advisory:
// writes can race the check, so the ALTER itself still performs the real conversion.
func preflightColumnType(data AlterColumnTypeWithCheckData, flatten bool) string {
ident := pgx.Identifier{data.TableName}
if data.SchemaName != "" {
if flatten {
ident = pgx.Identifier{data.SchemaName + "_" + data.TableName}
} else {
ident = pgx.Identifier{data.SchemaName, data.TableName}
}
}
table := ident.Sanitize()
body := fmt.Sprintf(`DECLARE current_type text;
BEGIN
SELECT pg_catalog.format_type(atttypid, atttypmod) INTO current_type
FROM pg_catalog.pg_attribute
WHERE attrelid = to_regclass(%s) AND attname = %s AND attnum > 0 AND NOT attisdropped;
IF current_type IS NOT NULL AND current_type <> ALL(ARRAY[%s]) THEN
`, sqlLiteral(table), sqlLiteral(data.ColumnName), data.EquivalentTypes)
if data.PreflightPredicate != "" {
body += fmt.Sprintf(` IF EXISTS (SELECT 1 FROM %s WHERE NOT COALESCE((%s), false)) THEN
RAISE EXCEPTION USING MESSAGE = %s;
END IF;
`, table, data.PreflightPredicate, sqlLiteral("type conversion preflight rejected rows for "+data.SchemaName+"."+data.TableName+"."+data.ColumnName))
}
// Aggregate evaluation forces the conversion for all rows, including non-first rows.
body += fmt.Sprintf(" PERFORM count((%s)::%s) FROM %s;\n END IF;\nEND;", data.UsingExpr, data.NewType, table)
return dollarBlock(body, "relspec_preflight")
}
func (w *Writer) generateTypePreflights(schema *models.Schema) ([]string, error) {
var statements []string
for _, table := range schema.Tables {
for _, col := range getSortedColumns(table.Columns) {
if col.Generated || col.Identity {
continue
}
data, err := conversionData(schema.Name, table.Name, col)
if err != nil {
return nil, err
}
statements = append(statements, preflightColumnType(data, w.options.FlattenSchema))
}
}
return statements, nil
}
+72
View File
@@ -0,0 +1,72 @@
package pgsql
import (
"strings"
"testing"
"git.warky.dev/wdevs/relspecgo/pkg/models"
"git.warky.dev/wdevs/relspecgo/pkg/writers"
)
func TestConversionMetadataValidationAndBlockDelimiters(t *testing.T) {
col := models.InitColumn("name", "accounts", "public")
col.Type = "integer"
for _, key := range []string{"pgsql_using", "pgsql_preflight"} {
for _, value := range []interface{}{true, " "} {
col.Metadata[key] = value
if _, err := conversionData("public", "accounts", col); err == nil || !strings.Contains(err.Error(), key) {
t.Fatalf("expected invalid %s metadata error: %v", key, err)
}
delete(col.Metadata, key)
}
}
col.Metadata["pgsql_using"] = "length('$relspec_0$ $relspec_preflight_0$')"
data, err := conversionData("public", "accounts", col)
if err != nil {
t.Fatal(err)
}
executor, err := NewTemplateExecutor(false)
if err != nil {
t.Fatal(err)
}
alter, err := executor.ExecuteAlterColumnTypeWithCheck(data)
if err != nil {
t.Fatal(err)
}
if !strings.HasPrefix(alter, "DO $relspec_1$") {
t.Fatalf("unsafe alter delimiter: %s", alter)
}
preflight := preflightColumnType(data, false)
if !strings.HasPrefix(preflight, "DO $relspec_preflight_1$") {
t.Fatalf("unsafe preflight delimiter: %s", preflight)
}
}
func TestPreflightsPrecedeAllSchemaMutations(t *testing.T) {
db := models.InitDatabase("preflight")
for _, name := range []string{"one", "two"} {
schema := models.InitSchema(name)
table := models.InitTable("accounts", name)
col := models.InitColumn("name", "accounts", name)
col.Type = "integer"
table.Columns[col.Name] = col
schema.Tables = append(schema.Tables, table)
db.Schemas = append(db.Schemas, schema)
}
stmts, err := NewWriter(&writers.WriterOptions{}).GenerateDatabaseStatements(db)
if err != nil {
t.Fatal(err)
}
lastPreflight, firstDDL := -1, len(stmts)
for i, stmt := range stmts {
if strings.HasPrefix(stmt, "DO $relspec_preflight") {
lastPreflight = i
}
if strings.HasPrefix(stmt, "CREATE SCHEMA") && i < firstDDL {
firstDDL = i
}
}
if lastPreflight < 0 || lastPreflight >= firstDDL {
t.Fatal("preflights must precede all schema changes")
}
}
File diff suppressed because it is too large Load Diff
+46 -62
View File
@@ -22,7 +22,7 @@ func TestWriteDatabase(t *testing.T) {
idCol := models.InitColumn("id", "users", "public") idCol := models.InitColumn("id", "users", "public")
idCol.Type = "integer" idCol.Type = "integer"
idCol.Description = "Primary key" idCol.Description = "Primary key"
idCol.Default = "nextval('public.identity_users_id'::regclass)" idCol.Default = "nextval('\"public\".\"identity_users_id\"'::regclass)"
table.Columns["id"] = idCol table.Columns["id"] = idCol
nameCol := models.InitColumn("name", "users", "public") nameCol := models.InitColumn("name", "users", "public")
@@ -117,7 +117,7 @@ func TestWriteDatabase_ConcurrentIndex(t *testing.T) {
output := buf.String() output := buf.String()
if !strings.Contains(output, "CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_users_email") { if !strings.Contains(output, "CREATE INDEX CONCURRENTLY IF NOT EXISTS \"idx_users_email\"") {
t.Errorf("Output missing CONCURRENTLY index creation:\n%s", output) t.Errorf("Output missing CONCURRENTLY index creation:\n%s", output)
} }
} }
@@ -151,7 +151,7 @@ func TestWriteDatabase_GinIndexOnTextArrayDoesNotUseTrigramOperatorClass(t *test
} }
output := buf.String() output := buf.String()
if !strings.Contains(output, `USING gin (tags array_ops)`) { if !strings.Contains(output, `USING gin ("tags" array_ops)`) {
t.Fatalf("expected GIN index on array column with array_ops, got:\n%s", output) t.Fatalf("expected GIN index on array column with array_ops, got:\n%s", output)
} }
if strings.Contains(output, "gin_trgm_ops") { if strings.Contains(output, "gin_trgm_ops") {
@@ -172,7 +172,7 @@ func TestWriteDatabase_GinIndexOnQuotedTextColumnUsesTrigramOperatorClass(t *tes
index := &models.Index{ index := &models.Index{
Name: "idx_agent_personas_name_gin", Name: "idx_agent_personas_name_gin",
Type: "gin", Type: "gin",
Columns: []string{`"name"`}, Columns: []string{"name"},
} }
table.Indexes[index.Name] = index table.Indexes[index.Name] = index
@@ -191,7 +191,7 @@ func TestWriteDatabase_GinIndexOnQuotedTextColumnUsesTrigramOperatorClass(t *tes
if !strings.Contains(output, `CREATE EXTENSION IF NOT EXISTS pg_trgm`) { if !strings.Contains(output, `CREATE EXTENSION IF NOT EXISTS pg_trgm`) {
t.Fatalf("expected trigram extension for text GIN index, got:\n%s", output) t.Fatalf("expected trigram extension for text GIN index, got:\n%s", output)
} }
if !strings.Contains(output, `USING gin (name gin_trgm_ops)`) { if !strings.Contains(output, `USING gin ("name" gin_trgm_ops)`) {
t.Fatalf("expected quoted text GIN index to include gin_trgm_ops, got:\n%s", output) t.Fatalf("expected quoted text GIN index to include gin_trgm_ops, got:\n%s", output)
} }
} }
@@ -225,7 +225,7 @@ func TestWriteDatabase_GinIndexOnJSONBUsesJSONBOperatorClass(t *testing.T) {
} }
output := buf.String() output := buf.String()
if !strings.Contains(output, `USING gin (details jsonb_ops)`) { if !strings.Contains(output, `USING gin ("details" jsonb_ops)`) {
t.Fatalf("expected GIN jsonb index to include jsonb_ops, got:\n%s", output) t.Fatalf("expected GIN jsonb index to include jsonb_ops, got:\n%s", output)
} }
if strings.Contains(output, "gin_trgm_ops") { if strings.Contains(output, "gin_trgm_ops") {
@@ -372,10 +372,10 @@ func TestWriteUniqueConstraints(t *testing.T) {
if !strings.Contains(output, "-- Unique constraints for schema: public") { if !strings.Contains(output, "-- Unique constraints for schema: public") {
t.Errorf("Output missing unique constraints header") t.Errorf("Output missing unique constraints header")
} }
if !strings.Contains(output, "ADD CONSTRAINT uq_email UNIQUE (email)") { if !strings.Contains(output, "ADD CONSTRAINT \"uq_email\" UNIQUE (\"email\")") {
t.Errorf("Output missing uq_email unique constraint\nFull output:\n%s", output) t.Errorf("Output missing uq_email unique constraint\nFull output:\n%s", output)
} }
if !strings.Contains(output, "ADD CONSTRAINT uq_guid UNIQUE (guid)") { if !strings.Contains(output, "ADD CONSTRAINT \"uq_guid\" UNIQUE (\"guid\")") {
t.Errorf("Output missing uq_guid unique constraint\nFull output:\n%s", output) t.Errorf("Output missing uq_guid unique constraint\nFull output:\n%s", output)
} }
} }
@@ -453,13 +453,13 @@ func TestWriteCheckConstraints(t *testing.T) {
if !strings.Contains(output, "-- Check constraints for schema: public") { if !strings.Contains(output, "-- Check constraints for schema: public") {
t.Errorf("Output missing check constraints header") t.Errorf("Output missing check constraints header")
} }
if !strings.Contains(output, "ADD CONSTRAINT ck_price_positive CHECK (price >= 0)") { if !strings.Contains(output, "ADD CONSTRAINT \"ck_price_positive\" CHECK (price >= 0)") {
t.Errorf("Output missing ck_price_positive check constraint\nFull output:\n%s", output) t.Errorf("Output missing ck_price_positive check constraint\nFull output:\n%s", output)
} }
if !strings.Contains(output, "ADD CONSTRAINT ck_status_valid CHECK (status IN ('active', 'inactive', 'discontinued'))") { if !strings.Contains(output, "ADD CONSTRAINT \"ck_status_valid\" CHECK (status IN ('active', 'inactive', 'discontinued'))") {
t.Errorf("Output missing ck_status_valid check constraint\nFull output:\n%s", output) t.Errorf("Output missing ck_status_valid check constraint\nFull output:\n%s", output)
} }
if !strings.Contains(output, "ADD CONSTRAINT ck_quantity_nonnegative CHECK (quantity >= 0)") { if !strings.Contains(output, "ADD CONSTRAINT \"ck_quantity_nonnegative\" CHECK (quantity >= 0)") {
t.Errorf("Output missing ck_quantity_nonnegative check constraint\nFull output:\n%s", output) t.Errorf("Output missing ck_quantity_nonnegative check constraint\nFull output:\n%s", output)
} }
} }
@@ -573,9 +573,9 @@ func TestWriteAllConstraintTypes(t *testing.T) {
// Verify all constraint types are present // Verify all constraint types are present
expectedConstraints := map[string]string{ expectedConstraints := map[string]string{
"Primary Key": "PRIMARY KEY", "Primary Key": "PRIMARY KEY",
"Unique": "ADD CONSTRAINT uq_order_number UNIQUE (order_number)", "Unique": "ADD CONSTRAINT \"uq_order_number\" UNIQUE (\"order_number\")",
"Check (total)": "ADD CONSTRAINT ck_total_positive CHECK (total > 0)", "Check (total)": "ADD CONSTRAINT \"ck_total_positive\" CHECK (total > 0)",
"Check (status)": "ADD CONSTRAINT ck_status_valid CHECK (status IN ('pending', 'completed', 'cancelled'))", "Check (status)": "ADD CONSTRAINT \"ck_status_valid\" CHECK (status IN ('pending', 'completed', 'cancelled'))",
"Foreign Key": "FOREIGN KEY", "Foreign Key": "FOREIGN KEY",
} }
@@ -640,7 +640,7 @@ func TestEscapeQuote(t *testing.T) {
input string input string
expected string expected string
}{ }{
{"simple text", "simple text"}, {"\"simple\" text", "\"simple\" text"},
{"text with 'quote'", "text with ''quote''"}, {"text with 'quote'", "text with ''quote''"},
{"multiple 'quotes' here", "multiple ''quotes'' here"}, {"multiple 'quotes' here", "multiple ''quotes'' here"},
{"", ""}, {"", ""},
@@ -1016,15 +1016,9 @@ func TestGenerateAddColumnStatements(t *testing.T) {
// Verify expected elements // Verify expected elements
expectedStrings := []string{ expectedStrings := []string{
"ALTER TABLE public.users ADD COLUMN id integer NOT NULL", "ALTER TABLE \"public\".\"users\" ADD COLUMN IF NOT EXISTS \"id\" integer NOT NULL",
"ALTER TABLE public.users ADD COLUMN email varchar(255) NOT NULL", "ALTER TABLE \"public\".\"users\" ADD COLUMN IF NOT EXISTS \"email\" varchar(255) NOT NULL",
"ALTER TABLE public.users ADD COLUMN status text DEFAULT 'active'", "ALTER TABLE \"public\".\"users\" ADD COLUMN IF NOT EXISTS \"status\" text DEFAULT 'active'",
"information_schema.columns",
"table_schema = 'public'",
"table_name = 'users'",
"column_name = 'id'",
"column_name = 'email'",
"column_name = 'status'",
} }
for _, expected := range expectedStrings { for _, expected := range expectedStrings {
@@ -1033,12 +1027,6 @@ func TestGenerateAddColumnStatements(t *testing.T) {
} }
} }
// Verify DO blocks are present for conditional adds
doBlockCount := strings.Count(output, "DO $$")
if doBlockCount < 3 {
t.Errorf("Expected at least 3 DO blocks (one per column), got %d", doBlockCount)
}
// Verify IF NOT EXISTS logic // Verify IF NOT EXISTS logic
ifNotExistsCount := strings.Count(output, "IF NOT EXISTS") ifNotExistsCount := strings.Count(output, "IF NOT EXISTS")
if ifNotExistsCount < 3 { if ifNotExistsCount < 3 {
@@ -1083,14 +1071,14 @@ func TestWriteAddColumnStatements(t *testing.T) {
t.Logf("Generated output:\n%s", output) t.Logf("Generated output:\n%s", output)
// Verify output contains expected elements // Verify output contains expected elements
if !strings.Contains(output, "ALTER TABLE public.products ADD COLUMN id integer") { if !strings.Contains(output, "ALTER TABLE \"public\".\"products\" ADD COLUMN IF NOT EXISTS \"id\" integer") {
t.Errorf("Output missing ADD COLUMN for id\nFull output:\n%s", output) t.Errorf("Output missing ADD COLUMN for id\nFull output:\n%s", output)
} }
if !strings.Contains(output, "ALTER TABLE public.products ADD COLUMN description text") { if !strings.Contains(output, "ALTER TABLE \"public\".\"products\" ADD COLUMN IF NOT EXISTS \"description\" text") {
t.Errorf("Output missing ADD COLUMN for description\nFull output:\n%s", output) t.Errorf("Output missing ADD COLUMN for description\nFull output:\n%s", output)
} }
if !strings.Contains(output, "DO $$") { if strings.Contains(output, "information_schema.columns") {
t.Errorf("Output missing DO block\nFull output:\n%s", output) t.Errorf("column additions should use native IF NOT EXISTS\nFull output:\n%s", output)
} }
} }
@@ -1127,16 +1115,16 @@ func TestWriteSchema_EmitsGuardedAlterColumnTypeStatements(t *testing.T) {
if !strings.Contains(output, "pg_catalog.format_type") { if !strings.Contains(output, "pg_catalog.format_type") {
t.Fatalf("expected guarded live-type check, got:\n%s", output) t.Fatalf("expected guarded live-type check, got:\n%s", output)
} }
if !strings.Contains(output, "ALTER COLUMN name TYPE character varying(255)") { if !strings.Contains(output, "ALTER COLUMN \"name\" TYPE character varying(255)") {
t.Fatalf("expected guarded alter for character varying(255), got:\n%s", output) t.Fatalf("expected guarded alter for character varying(255), got:\n%s", output)
} }
if !strings.Contains(output, "ARRAY['varchar(255)', 'character varying(255)']") { if !strings.Contains(output, "ARRAY['varchar(255)', 'character varying(255)']") {
t.Fatalf("expected equivalent type spellings for varchar guard, got:\n%s", output) t.Fatalf("expected equivalent type spellings for varchar guard, got:\n%s", output)
} }
if !strings.Contains(output, "ALTER COLUMN tags TYPE text[]") { if !strings.Contains(output, "ALTER COLUMN \"tags\" TYPE text[]") {
t.Fatalf("expected guarded alter for array type, got:\n%s", output) t.Fatalf("expected guarded alter for array type, got:\n%s", output)
} }
if !strings.Contains(output, `ALTER COLUMN tags TYPE text[] USING tags::text[];`) { if !strings.Contains(output, `ALTER COLUMN "tags" TYPE text[] USING "tags"::text[];`) {
t.Fatalf("expected guarded alter for array type to include USING cast, got:\n%s", output) t.Fatalf("expected guarded alter for array type to include USING cast, got:\n%s", output)
} }
} }
@@ -1170,7 +1158,7 @@ func TestWriteSchema_EmitsGuardedAlterColumnDefaultStatements(t *testing.T) {
if !strings.Contains(output, "pg_get_expr(d.adbin, d.adrelid)") { if !strings.Contains(output, "pg_get_expr(d.adbin, d.adrelid)") {
t.Fatalf("expected guarded live-default check, got:\n%s", output) t.Fatalf("expected guarded live-default check, got:\n%s", output)
} }
if !strings.Contains(output, "ALTER COLUMN status SET DEFAULT 'active'") { if !strings.Contains(output, "ALTER COLUMN \"status\" SET DEFAULT 'active'") {
t.Fatalf("expected guarded SET DEFAULT for status column, got:\n%s", output) t.Fatalf("expected guarded SET DEFAULT for status column, got:\n%s", output)
} }
} }
@@ -1201,12 +1189,12 @@ func TestWriteSchema_AlterColumnDefaultStripsBackticksFromFunctionExpression(t *
if strings.Contains(output, "`") { if strings.Contains(output, "`") {
t.Fatalf("expected no backticks in generated SQL, got:\n%s", output) t.Fatalf("expected no backticks in generated SQL, got:\n%s", output)
} }
if !strings.Contains(output, "ALTER COLUMN updatedat SET DEFAULT now()") { if !strings.Contains(output, "ALTER COLUMN \"updatedat\" SET DEFAULT now()") {
t.Fatalf("expected guarded SET DEFAULT now() without backticks, got:\n%s", output) t.Fatalf("expected guarded SET DEFAULT now() without backticks, got:\n%s", output)
} }
} }
func TestWriteSchema_GuardedAlterColumnTypeFallsBackOnConversionFailure(t *testing.T) { func TestWriteSchema_GuardedAlterColumnTypePreservesConversionFailure(t *testing.T) {
db := models.InitDatabase("testdb") db := models.InitDatabase("testdb")
schema := models.InitSchema("public") schema := models.InitSchema("public")
@@ -1228,17 +1216,13 @@ func TestWriteSchema_GuardedAlterColumnTypeFallsBackOnConversionFailure(t *testi
} }
output := buf.String() output := buf.String()
if !strings.Contains(output, "EXCEPTION WHEN OTHERS THEN") { if !strings.Contains(output, "ALTER COLUMN \"name\" TYPE integer USING \"name\"::integer;") {
t.Fatalf("expected guarded alter to fall back on conversion failure, got:\n%s", output) t.Fatalf("expected explicit type conversion, got:\n%s", output)
} }
if !strings.Contains(output, "renamed_column := 'name_' || trim(both '_' from regexp_replace(lower(current_type)") { for _, unsafe := range []string{"EXCEPTION WHEN OTHERS", "renamed_column", "RENAME COLUMN"} {
t.Fatalf("expected fallback to derive a renamed column name from the live type, got:\n%s", output) if strings.Contains(output, unsafe) {
t.Fatalf("type conversion must propagate errors without replacing the column (%s):\n%s", unsafe, output)
} }
if !strings.Contains(output, "RENAME COLUMN name TO %I") {
t.Fatalf("expected fallback to rename the existing column, got:\n%s", output)
}
if !strings.Contains(output, "ADD COLUMN name integer") {
t.Fatalf("expected fallback to add a fresh column with the new type, got:\n%s", output)
} }
} }
@@ -1274,7 +1258,7 @@ func TestWriteSchema_EmitsGuardedAlterColumnNullabilityStatements(t *testing.T)
if !strings.Contains(output, "current_not_null IS DISTINCT FROM false") { if !strings.Contains(output, "current_not_null IS DISTINCT FROM false") {
t.Fatalf("expected guard comparing live nullability against desired value, got:\n%s", output) t.Fatalf("expected guard comparing live nullability against desired value, got:\n%s", output)
} }
if !strings.Contains(output, "ALTER COLUMN rid_service_instance_type DROP NOT NULL") { if !strings.Contains(output, "ALTER COLUMN \"rid_service_instance_type\" DROP NOT NULL") {
t.Fatalf("expected guarded DROP NOT NULL for nullable column, got:\n%s", output) t.Fatalf("expected guarded DROP NOT NULL for nullable column, got:\n%s", output)
} }
} }
@@ -1300,13 +1284,13 @@ func TestWriteSchema_UsesStorageTypeForSerialAlterStatements(t *testing.T) {
} }
output := buf.String() output := buf.String()
if !strings.Contains(output, "ALTER COLUMN id TYPE bigint") { if !strings.Contains(output, "ALTER COLUMN \"id\" TYPE bigint") {
t.Fatalf("expected serial alter to use bigint storage type, got:\n%s", output) t.Fatalf("expected serial alter to use bigint storage type, got:\n%s", output)
} }
if strings.Contains(output, "ALTER COLUMN id TYPE bigserial;") { if strings.Contains(output, "ALTER COLUMN \"id\" TYPE bigserial;") {
t.Fatalf("did not expect invalid bigserial alter statement, got:\n%s", output) t.Fatalf("did not expect invalid bigserial alter statement, got:\n%s", output)
} }
if !strings.Contains(output, `ALTER COLUMN id TYPE bigint USING id::bigint;`) { if !strings.Contains(output, `ALTER COLUMN "id" TYPE bigint USING "id"::bigint;`) {
t.Fatalf("expected serial alter to include USING cast, got:\n%s", output) t.Fatalf("expected serial alter to include USING cast, got:\n%s", output)
} }
} }
@@ -1377,7 +1361,7 @@ func TestWriteDatabase_VectorAndPostGISColumnsCreateExtensions(t *testing.T) {
func TestWriteDatabase_HNSWIndexUsesDefaultVectorOperatorClass(t *testing.T) { func TestWriteDatabase_HNSWIndexUsesDefaultVectorOperatorClass(t *testing.T) {
output := writeDatabaseOutput(t, buildVectorSpatialSchema("hnsw", "")) output := writeDatabaseOutput(t, buildVectorSpatialSchema("hnsw", ""))
if !strings.Contains(output, "USING hnsw (embedding vector_cosine_ops)") { if !strings.Contains(output, "USING hnsw (\"embedding\" vector_cosine_ops)") {
t.Fatalf("expected hnsw index with default vector operator class, got:\n%s", output) t.Fatalf("expected hnsw index with default vector operator class, got:\n%s", output)
} }
if !strings.Contains(output, "CREATE EXTENSION IF NOT EXISTS vector;") { if !strings.Contains(output, "CREATE EXTENSION IF NOT EXISTS vector;") {
@@ -1388,7 +1372,7 @@ func TestWriteDatabase_HNSWIndexUsesDefaultVectorOperatorClass(t *testing.T) {
func TestWriteDatabase_VectorIndexHonoursRequestedOperatorClassAndStorageParameters(t *testing.T) { func TestWriteDatabase_VectorIndexHonoursRequestedOperatorClassAndStorageParameters(t *testing.T) {
output := writeDatabaseOutput(t, buildVectorSpatialSchema("ivfflat", "opclass=vector_l2_ops; with (lists=100)")) output := writeDatabaseOutput(t, buildVectorSpatialSchema("ivfflat", "opclass=vector_l2_ops; with (lists=100)"))
if !strings.Contains(output, "USING ivfflat (embedding vector_l2_ops) WITH (lists = 100)") { if !strings.Contains(output, "USING ivfflat (\"embedding\" vector_l2_ops) WITH (lists = 100)") {
t.Fatalf("expected ivfflat index with requested opclass and storage parameters, got:\n%s", output) t.Fatalf("expected ivfflat index with requested opclass and storage parameters, got:\n%s", output)
} }
} }
@@ -1396,7 +1380,7 @@ func TestWriteDatabase_VectorIndexHonoursRequestedOperatorClassAndStorageParamet
func TestWriteDatabase_VectorIndexIgnoresIncompatibleOperatorClass(t *testing.T) { func TestWriteDatabase_VectorIndexIgnoresIncompatibleOperatorClass(t *testing.T) {
output := writeDatabaseOutput(t, buildVectorSpatialSchema("hnsw", "opclass=halfvec_l2_ops")) output := writeDatabaseOutput(t, buildVectorSpatialSchema("hnsw", "opclass=halfvec_l2_ops"))
if !strings.Contains(output, "USING hnsw (embedding vector_cosine_ops)") { if !strings.Contains(output, "USING hnsw (\"embedding\" vector_cosine_ops)") {
t.Fatalf("expected halfvec operator class to be rejected for a vector column, got:\n%s", output) t.Fatalf("expected halfvec operator class to be rejected for a vector column, got:\n%s", output)
} }
} }
@@ -1432,7 +1416,7 @@ func TestWriteDatabase_GistIndexOnGeometryUsesDefaultOperatorClass(t *testing.T)
output := writeDatabaseOutput(t, db) output := writeDatabaseOutput(t, db)
if !strings.Contains(output, "USING gist (geom)") { if !strings.Contains(output, "USING gist (\"geom\")") {
t.Fatalf("expected gist index to rely on the PostGIS default operator class, got:\n%s", output) t.Fatalf("expected gist index to rely on the PostGIS default operator class, got:\n%s", output)
} }
} }
@@ -1458,7 +1442,7 @@ func TestWriteDatabase_GistIndexHonoursRequestedSpatialOperatorClass(t *testing.
output := writeDatabaseOutput(t, db) output := writeDatabaseOutput(t, db)
if !strings.Contains(output, "USING gist (geom gist_geometry_ops_nd)") { if !strings.Contains(output, "USING gist (\"geom\" gist_geometry_ops_nd)") {
t.Fatalf("expected requested spatial operator class, got:\n%s", output) t.Fatalf("expected requested spatial operator class, got:\n%s", output)
} }
} }
@@ -1476,7 +1460,7 @@ func TestGenerateDatabaseStatements_VectorIndexIncludesOperatorClassAndParameter
for _, want := range []string{ for _, want := range []string{
"CREATE EXTENSION IF NOT EXISTS vector", "CREATE EXTENSION IF NOT EXISTS vector",
"CREATE EXTENSION IF NOT EXISTS postgis", "CREATE EXTENSION IF NOT EXISTS postgis",
"USING hnsw (embedding vector_ip_ops) WITH (m = 16)", "USING hnsw (\"embedding\" vector_ip_ops) WITH (m = 16)",
} { } {
if !strings.Contains(joined, want) { if !strings.Contains(joined, want) {
t.Fatalf("expected statements to contain %q, got:\n%s", want, joined) t.Fatalf("expected statements to contain %q, got:\n%s", want, joined)
@@ -1525,7 +1509,7 @@ func TestGenerateColumnDefinition_GeneratedColumnEmitsGeneratedClauseNotDefault(
got := w.generateColumnDefinition(col) got := w.generateColumnDefinition(col)
want := "geom text GENERATED ALWAYS AS (st_makepoint(lon, lat)) STORED NOT NULL" want := "\"geom\" text GENERATED ALWAYS AS (st_makepoint(lon, lat)) STORED NOT NULL"
if got != want { if got != want {
t.Fatalf("generateColumnDefinition() = %q, want %q", got, want) t.Fatalf("generateColumnDefinition() = %q, want %q", got, want)
} }
@@ -1544,7 +1528,7 @@ func TestGenerateColumnDefinition_IdentityColumnEmitsIdentityClauseNotDefault(t
got := w.generateColumnDefinition(col) got := w.generateColumnDefinition(col)
want := "id bigint GENERATED ALWAYS AS IDENTITY NOT NULL" want := "\"id\" bigint GENERATED ALWAYS AS IDENTITY NOT NULL"
if got != want { if got != want {
t.Fatalf("generateColumnDefinition() = %q, want %q", got, want) t.Fatalf("generateColumnDefinition() = %q, want %q", got, want)
} }
@@ -1591,7 +1575,7 @@ func TestWriteDatabase_MultipleForeignKeysToSameTable(t *testing.T) {
} }
output := buf.String() output := buf.String()
for _, name := range []string{"rid_department", "rid_manager"} { for _, name := range []string{"rid_department", "rid_manager"} {
want := "ADD CONSTRAINT fk_employee_" + name + "\n FOREIGN KEY (" + name + ")" want := "ADD CONSTRAINT " + quoteIdent("fk_employee_"+name) + "\n FOREIGN KEY (" + quoteIdent(name) + ")"
if !strings.Contains(output, want) { if !strings.Contains(output, want) {
t.Fatalf("run %d: missing %q in output:\n%s", i, want, output) t.Fatalf("run %d: missing %q in output:\n%s", i, want, output)
} }
+32 -5
View File
@@ -186,11 +186,30 @@ Future support planned for:
- SQLite - SQLite
- Generic SQL via database/sql - Generic SQL via database/sql
## Transaction Behavior ## Execution and transaction behavior
**Current**: Each script executes in its own implicit transaction (PostgreSQL default behavior) This is a raw PostgreSQL script runner, not a migration manager. It has no applied
history, checksums, migration lock, or duplicate-run protection. Re-running repeats
the supplied scripts. Scripts may contain transaction control, so RelSpec does not
wrap them in a transaction and rejects `WriterOptions.AtomicExecution`.
**Future Enhancement**: Option to wrap all scripts in a single transaction for atomic execution with rollback on error. By default execution stops on the first SQL or embed-preparation failure. Set
`WriterOptions.ContinueOnError`, legacy `Metadata["ignore_errors"]`, or CLI
`--ignore-errors` / `--continue-on-error` / `--coe` to attempt later scripts. Any
failure still returns a nonzero result and prevents dependent jobs from running.
The shared `writers.ExecutionReport` is available at `Metadata["execution_report"]`;
`Metadata["report_path"]` requests a JSON report and report-write failures are errors.
Counters and operations cover every schema in the invocation, including scripts
skipped after stopping or because they are empty. Legacy `execution_total`,
`execution_success`, `execution_failed`, `execution_skipped`, and
`execution_rolled_back` metadata counters are also populated on failure.
Reports use `outcome: script_managed` and successful script status `executed`, rather
than claiming committed database changes. A script may commit, roll back, or leave a
transaction open, and a failed script may already have committed work. The report
records which scripts succeeded/failed/skipped; it cannot infer statement-level
commit or rollback inside arbitrary scripts. The connection closes after execution.
## Performance Considerations ## Performance Considerations
@@ -226,7 +245,7 @@ docker run -d --name postgres-test \
postgres:16 postgres:16
# Run your integration tests # Run your integration tests
go test -tags=integration ./pkg/writers/sqlexec/ RELSPEC_TEST_PG_CONN='postgres://postgres:test@127.0.0.1:5432/testdb?sslmode=disable' go test -count=1 ./pkg/writers/sqlexec/
# Cleanup # Cleanup
docker stop postgres-test docker stop postgres-test
@@ -237,7 +256,7 @@ docker rm postgres-test
- `WriteTable()` is not supported (returns error) - `WriteTable()` is not supported (returns error)
- Requires PostgreSQL connection (no offline mode) - Requires PostgreSQL connection (no offline mode)
- No built-in transaction wrapping (yet) - No executor-level transaction wrapping for raw scripts
- No rollback script support (yet, though `models.Script.Rollback` field exists) - No rollback script support (yet, though `models.Script.Rollback` field exists)
## Related ## Related
@@ -245,3 +264,11 @@ docker rm postgres-test
- **SQL Directory Reader**: `pkg/readers/sqldir/` - Read scripts from filesystem - **SQL Directory Reader**: `pkg/readers/sqldir/` - Read scripts from filesystem
- **Script Model**: `pkg/models/models.go` - Script structure definition - **Script Model**: `pkg/models/models.go` - Script structure definition
- **pgx Documentation**: https://github.com/jackc/pgx - PostgreSQL driver docs - **pgx Documentation**: https://github.com/jackc/pgx - PostgreSQL driver docs
## Model embed access
SQL directory readers expand embed directives within their trusted input root.
When executing unexpanded directives from a model, explicitly configure
`WriterOptions.Metadata["asset_root"]` with the allowed directory. Script
`source_path` metadata alone never authorizes file reads. Paths, symlinks, regular
file types, and the 64 MiB asset-size limit are checked before reading.
+66 -120
View File
@@ -42,22 +42,25 @@ func (w *Writer) WriteDatabase(db *models.Database) error {
return fmt.Errorf("connection_string is required in writer metadata") return fmt.Errorf("connection_string is required in writer metadata")
} }
// Preserve schema order and script ordering within each schema, but aggregate once.
var scripts []*models.Script
for _, schema := range db.Schemas {
for _, script := range sortedScripts(schema.Scripts) {
scriptCopy := *script
scriptCopy.Name = schema.Name + "." + script.Name
scripts = append(scripts, &scriptCopy)
}
}
// Connect to database // Connect to database
ctx := context.Background() ctx := context.Background()
conn, err := pgsql.Connect(ctx, connString, "writer-sqlexec") conn, err := pgsql.Connect(ctx, connString, "writer-sqlexec")
if err != nil { if err != nil {
return fmt.Errorf("failed to connect to database: %w", err) return writers.FailExecution(w.options, "PostgreSQL scripts", scriptOperations(scripts), fmt.Errorf("failed to connect to database: %w", err))
} }
defer conn.Close(ctx) defer conn.Close(ctx)
// Execute scripts from all schemas return w.runScripts(ctx, conn, scripts)
for _, schema := range db.Schemas {
if err := w.executeScripts(ctx, conn, schema.Scripts); err != nil {
return fmt.Errorf("failed to execute scripts from schema %s: %w", schema.Name, err)
}
}
return nil
} }
// WriteSchema executes all scripts from a single schema // WriteSchema executes all scripts from a single schema
@@ -76,7 +79,7 @@ func (w *Writer) WriteSchema(schema *models.Schema) error {
ctx := context.Background() ctx := context.Background()
conn, err := pgsql.Connect(ctx, connString, "writer-sqlexec") conn, err := pgsql.Connect(ctx, connString, "writer-sqlexec")
if err != nil { if err != nil {
return fmt.Errorf("failed to connect to database: %w", err) return writers.FailExecution(w.options, "PostgreSQL scripts", scriptOperations(sortedScripts(schema.Scripts)), fmt.Errorf("failed to connect to database: %w", err))
} }
defer conn.Close(ctx) defer conn.Close(ctx)
@@ -93,119 +96,62 @@ func (w *Writer) WriteTable(table *models.Table) error {
return fmt.Errorf("WriteTable is not supported for SQL script execution") return fmt.Errorf("WriteTable is not supported for SQL script execution")
} }
// executeScripts executes scripts in Priority, Sequence, then Name order // executeScripts executes scripts in Priority, Sequence, then Name order.
func (w *Writer) executeScripts(ctx context.Context, conn *pgx.Conn, scripts []*models.Script) error { func (w *Writer) executeScripts(ctx context.Context, conn *pgx.Conn, scripts []*models.Script) error {
if len(scripts) == 0 { return w.runScripts(ctx, conn, sortedScripts(scripts))
return nil
}
// Check if we should ignore errors
ignoreErrors := false
if val, ok := w.options.Metadata["ignore_errors"].(bool); ok {
ignoreErrors = val
}
// Track failed scripts and execution counts
var failedScripts []struct {
name string
priority int
sequence uint
err error
}
successCount := 0
totalCount := 0
// Sort scripts by Priority (ascending), Sequence (ascending), then Name (ascending)
sortedScripts := make([]*models.Script, len(scripts))
copy(sortedScripts, scripts)
sort.Slice(sortedScripts, func(i, j int) bool {
if sortedScripts[i].Priority != sortedScripts[j].Priority {
return sortedScripts[i].Priority < sortedScripts[j].Priority
}
if sortedScripts[i].Sequence != sortedScripts[j].Sequence {
return sortedScripts[i].Sequence < sortedScripts[j].Sequence
}
return sortedScripts[i].Name < sortedScripts[j].Name
})
// Execute each script in order
for _, script := range sortedScripts {
if script.SQL == "" {
continue
}
totalCount++
fmt.Printf("Executing script: %s (Priority=%d, Sequence=%d)\n",
script.Name, script.Priority, script.Sequence)
// Execute the SQL script
sql, err := processEmbedDirectives(script)
if err != nil {
if ignoreErrors {
fmt.Printf("⚠ Error preparing %s: %v (continuing due to --ignore-errors)\n", script.Name, err)
failedScripts = append(failedScripts, struct {
name string
priority int
sequence uint
err error
}{
name: script.Name,
priority: script.Priority,
sequence: script.Sequence,
err: err,
})
continue
}
return fmt.Errorf("script %s (Priority=%d, Sequence=%d): %w",
script.Name, script.Priority, script.Sequence, err)
}
_, err = conn.Exec(ctx, sql)
if err != nil {
if ignoreErrors {
fmt.Printf("⚠ Error executing %s: %v (continuing due to --ignore-errors)\n", script.Name, err)
failedScripts = append(failedScripts, struct {
name string
priority int
sequence uint
err error
}{
name: script.Name,
priority: script.Priority,
sequence: script.Sequence,
err: err,
})
continue
}
return fmt.Errorf("script %s (Priority=%d, Sequence=%d): %w",
script.Name, script.Priority, script.Sequence, err)
}
successCount++
fmt.Printf("✓ Successfully executed: %s\n", script.Name)
}
// Store execution results in metadata for caller
w.options.Metadata["execution_total"] = totalCount
w.options.Metadata["execution_success"] = successCount
w.options.Metadata["execution_failed"] = len(failedScripts)
// Print summary of failed scripts if any
if len(failedScripts) > 0 {
fmt.Printf("\n⚠ Failed Scripts Summary (%d failed):\n", len(failedScripts))
for i, failed := range failedScripts {
fmt.Printf(" %d. %s (Priority=%d, Sequence=%d)\n Error: %v\n",
i+1, failed.name, failed.priority, failed.sequence, failed.err)
}
}
return nil
} }
func processEmbedDirectives(script *models.Script) (string, error) { func sortedScripts(scripts []*models.Script) []*models.Script {
sqlPath, _ := script.Metadata[assetloader.ScriptSourcePathMetadataKey].(string) ordered := append([]*models.Script(nil), scripts...)
if sqlPath == "" { sort.SliceStable(ordered, func(i, j int) bool {
if ordered[i].Priority != ordered[j].Priority {
return ordered[i].Priority < ordered[j].Priority
}
if ordered[i].Sequence != ordered[j].Sequence {
return ordered[i].Sequence < ordered[j].Sequence
}
return ordered[i].Name < ordered[j].Name
})
return ordered
}
func (w *Writer) runScripts(ctx context.Context, conn *pgx.Conn, scripts []*models.Script) error {
operations := scriptOperations(scripts)
backend := writers.ExecutionBackend{ScriptManaged: true, Exec: func(ctx context.Context, i int, _ string) error {
script := scripts[i]
fmt.Printf("Executing script: %s (Priority=%d, Sequence=%d)\n", script.Name, script.Priority, script.Sequence)
assetRoot, _ := w.options.Metadata["asset_root"].(string)
sql, err := processEmbedDirectives(script, assetRoot)
if err != nil {
return fmt.Errorf("preparing script %s: %w", script.Name, err)
}
if _, err := conn.Exec(ctx, sql); err != nil {
return fmt.Errorf("script %s: %w", script.Name, err)
}
return nil
}}
// Supplied scripts can manage transactions themselves. Do not promise whole-script rollback.
return writers.RunExecution(ctx, w.options, &writers.ExecutionReport{Dialect: "PostgreSQL scripts"}, operations, backend)
}
func scriptOperations(scripts []*models.Script) []writers.ExecutionOperation {
operations := make([]writers.ExecutionOperation, len(scripts))
for i, script := range scripts {
operations[i] = writers.ExecutionOperation{Name: script.Name, Statement: script.SQL}
}
return operations
}
func processEmbedDirectives(script *models.Script, assetRoot string) (string, error) {
if !assetloader.HasEmbedDirectives(script.SQL) {
return script.SQL, nil return script.SQL, nil
} }
return assetloader.ProcessEmbedDirectives(sqlPath, script.SQL) if assetRoot == "" {
return "", fmt.Errorf("asset_root is required to resolve embed directives from a model; source_path metadata does not authorize file access")
}
sqlPath, _ := script.Metadata[assetloader.ScriptSourcePathMetadataKey].(string)
if sqlPath == "" {
return "", fmt.Errorf("source_path is required for model embed directives")
}
return assetloader.ProcessEmbedDirectivesWithOptions(sqlPath, script.SQL, assetloader.EmbedOptions{AssetRoot: assetRoot})
} }
+29 -6
View File
@@ -2,6 +2,7 @@ package sqlexec
import ( import (
"context" "context"
"errors"
"fmt" "fmt"
"os" "os"
"path/filepath" "path/filepath"
@@ -121,7 +122,7 @@ func TestLive_ExecuteScriptsOrder(t *testing.T) {
if strings.Join(got, ",") != strings.Join(want, ",") { if strings.Join(got, ",") != strings.Join(want, ",") {
t.Errorf("execution order = %v, want %v", got, want) t.Errorf("execution order = %v, want %v", got, want)
} }
if opts.Metadata["execution_total"] != 6 || opts.Metadata["execution_success"] != 6 || opts.Metadata["execution_failed"] != 0 { if opts.Metadata["execution_total"] != 7 || opts.Metadata["execution_success"] != 6 || opts.Metadata["execution_failed"] != 0 {
t.Errorf("counts: %v", opts.Metadata) t.Errorf("counts: %v", opts.Metadata)
} }
} }
@@ -160,8 +161,8 @@ func TestLive_IgnoreErrorsContinues(t *testing.T) {
{Name: "02_ok", Priority: 2, SQL: fmt.Sprintf("CREATE TABLE %s.after(id int);", schema)}, {Name: "02_ok", Priority: 2, SQL: fmt.Sprintf("CREATE TABLE %s.after(id int);", schema)},
} }
opts := liveOptions(connString, map[string]interface{}{"ignore_errors": true}) opts := liveOptions(connString, map[string]interface{}{"ignore_errors": true})
if err := NewWriter(opts).WriteSchema(&models.Schema{Name: schema, Scripts: scripts}); err != nil { if err := NewWriter(opts).WriteSchema(&models.Schema{Name: schema, Scripts: scripts}); err == nil {
t.Fatalf("ignore_errors must not fail: %v", err) t.Fatal("continuation must still return failure")
} }
if opts.Metadata["execution_total"] != 2 || opts.Metadata["execution_success"] != 1 || opts.Metadata["execution_failed"] != 1 { if opts.Metadata["execution_total"] != 2 || opts.Metadata["execution_success"] != 1 || opts.Metadata["execution_failed"] != 1 {
t.Errorf("counts: %v", opts.Metadata) t.Errorf("counts: %v", opts.Metadata)
@@ -186,8 +187,8 @@ func TestLive_EmbedDirectiveErrorHandling(t *testing.T) {
} }
opts := liveOptions(connString, map[string]interface{}{"ignore_errors": true}) opts := liveOptions(connString, map[string]interface{}{"ignore_errors": true})
if err := NewWriter(opts).WriteSchema(&models.Schema{Name: schema, Scripts: []*models.Script{bad}}); err != nil { if err := NewWriter(opts).WriteSchema(&models.Schema{Name: schema, Scripts: []*models.Script{bad}}); err == nil {
t.Errorf("ignore_errors: %v", err) t.Error("embed preparation failures must propagate even when continuing")
} }
if opts.Metadata["execution_failed"] != 1 { if opts.Metadata["execution_failed"] != 1 {
t.Errorf("counts: %v", opts.Metadata) t.Errorf("counts: %v", opts.Metadata)
@@ -217,7 +218,29 @@ func TestLive_WriteDatabaseMultiSchema(t *testing.T) {
// A failure in one schema aborts and names that schema. // A failure in one schema aborts and names that schema.
db.Schemas[1].Scripts[0].SQL = "SELECT * FROM definitely_missing_table;" db.Schemas[1].Scripts[0].SQL = "SELECT * FROM definitely_missing_table;"
err := NewWriter(liveOptions(connString, nil)).WriteDatabase(db) err := NewWriter(liveOptions(connString, nil)).WriteDatabase(db)
if err == nil || !strings.Contains(err.Error(), "schema "+s2) { if err == nil || !strings.Contains(err.Error(), s2+".b") {
t.Errorf("expected error naming schema %s, got %v", s2, err) t.Errorf("expected error naming schema %s, got %v", s2, err)
} }
} }
func TestLive_MultiSchemaFailuresAggregate(t *testing.T) {
connection := liveConn(t)
opts := liveOptions(connection, nil)
opts.ContinueOnError = true
db := &models.Database{Schemas: []*models.Schema{
{Name: "one", Scripts: []*models.Script{{Name: "bad", SQL: "SELECT 1 / 0"}, {Name: "empty"}}},
{Name: "two", Scripts: []*models.Script{{Name: "ok", SQL: "SELECT 1"}}},
}}
err := NewWriter(opts).WriteDatabase(db)
var failure *writers.ExecutionFailure
if !errors.As(err, &failure) {
t.Fatalf("expected structured failure: %v", err)
}
report := failure.Report
if report.TotalStatements != 3 || report.FailedStatements != 1 || report.ExecutedStatements != 1 || report.SkippedStatements != 1 || report.Outcome != "script_managed" {
t.Fatalf("incorrect aggregate: %+v", report)
}
if report.Operations[0].Name != "one.bad" || report.Operations[2].Name != "two.ok" || report.Operations[2].Status != "executed" {
t.Fatalf("script details: %+v", report.Operations)
}
}
+20 -7
View File
@@ -232,7 +232,7 @@ func TestProcessEmbedDirectives_UsesScriptSourcePath(t *testing.T) {
script.SQL = "-- @embed: path=body.txt var=:body mode=text\nSELECT :body;" script.SQL = "-- @embed: path=body.txt var=:body mode=text\nSELECT :body;"
script.Metadata[assetloader.ScriptSourcePathMetadataKey] = sqlPath script.Metadata[assetloader.ScriptSourcePathMetadataKey] = sqlPath
got, err := processEmbedDirectives(script) got, err := processEmbedDirectives(script, dir)
if err != nil { if err != nil {
t.Fatalf("processEmbedDirectives failed: %v", err) t.Fatalf("processEmbedDirectives failed: %v", err)
} }
@@ -241,15 +241,28 @@ func TestProcessEmbedDirectives_UsesScriptSourcePath(t *testing.T) {
} }
} }
func TestProcessEmbedDirectives_NoSourcePathLeavesSQLUnchanged(t *testing.T) { func TestProcessEmbedDirectives_NoSourcePathCannotAuthorizeRead(t *testing.T) {
script := models.InitScript("seed") script := models.InitScript("seed")
script.SQL = "-- @embed: path=missing.txt var=:body mode=text\nSELECT :body;" script.SQL = "-- @embed: path=missing.txt var=:body mode=text\nSELECT :body;"
got, err := processEmbedDirectives(script) got, err := processEmbedDirectives(script, t.TempDir())
if err != nil { if err == nil || got != "" {
t.Fatalf("processEmbedDirectives failed: %v", err) t.Fatalf("expected missing source_path failure, got %q %v", got, err)
} }
if got != script.SQL { }
t.Fatalf("expected unchanged SQL without source path, got:\n%s", got)
func TestProcessEmbedDirectivesModelSourcePathIsNotAuthorization(t *testing.T) {
dir := t.TempDir()
if err := os.WriteFile(filepath.Join(dir, "secret.txt"), []byte("secret"), 0o600); err != nil {
t.Fatal(err)
}
script := models.InitScript("untrusted")
script.SQL = "-- @embed: path=secret.txt var=:body mode=text\nSELECT :body;"
script.Metadata[assetloader.ScriptSourcePathMetadataKey] = filepath.Join(dir, "fake.sql")
if _, err := processEmbedDirectives(script, ""); err == nil || !strings.Contains(err.Error(), "asset_root is required") {
t.Fatalf("source metadata must not authorize file read: %v", err)
}
if _, err := processEmbedDirectives(script, t.TempDir()); err == nil {
t.Fatal("source path outside trusted root must fail")
} }
} }
+28 -29
View File
@@ -117,50 +117,49 @@ func (c *statementCollector) Write(p []byte) (int, error) {
// against the SQLite database file at dbPath. // against the SQLite database file at dbPath.
func (w *Writer) executeDatabaseSQL(db *models.Database, dbPath string) error { func (w *Writer) executeDatabaseSQL(db *models.Database, dbPath string) error {
collector := &statementCollector{} collector := &statementCollector{}
saved := w.writer
w.writer = collector w.writer = collector
defer func() { w.writer = saved }()
if err := w.writeContent(db); err != nil { if err := w.writeContent(db); err != nil {
return fmt.Errorf("failed to generate SQL statements: %w", err) return fmt.Errorf("failed to generate SQL statements: %w", err)
} }
conn, err := sql.Open("sqlite", dbPath) conn, err := sql.Open("sqlite", dbPath)
if err != nil { if err != nil {
return fmt.Errorf("failed to open sqlite database %q: %w", dbPath, err) return writers.FailExecution(w.options, "SQLite", writers.StatementOperations(collector.statements), fmt.Errorf("failed to open sqlite database %q: %w", dbPath, err))
} }
defer conn.Close() defer conn.Close()
// Pin one connection so connection-local settings (including foreign_keys) persist.
ctx := context.Background() ctx := context.Background()
ignoreErrors := false session, err := conn.Conn(ctx)
if val, ok := w.options.Metadata["ignore_errors"].(bool); ok { if err != nil {
ignoreErrors = val return writers.FailExecution(w.options, "SQLite", writers.StatementOperations(collector.statements), err)
} }
defer session.Close()
total, executed := 0, 0 var tx *sql.Tx
var execErrors []string backend := writers.ExecutionBackend{
for _, stmt := range collector.statements { Exec: func(ctx context.Context, _ int, stmt string) error {
if strings.HasPrefix(stmt, "--") { if tx != nil {
continue _, err := tx.ExecContext(ctx, stmt)
return err
} }
_, err := session.ExecContext(ctx, stmt)
total++ return err
if _, err := conn.ExecContext(ctx, stmt); err != nil { },
execErrors = append(execErrors, fmt.Sprintf("statement %d (%s): %v", total, truncateStatement(stmt), err)) Begin: func(ctx context.Context) error {
if !ignoreErrors { // SQLite ignores foreign_keys changes inside transactions.
break if _, err := session.ExecContext(ctx, "PRAGMA foreign_keys=ON"); err != nil {
return err
} }
continue var err error
tx, err = session.BeginTx(ctx, nil)
return err
},
Commit: func(context.Context) error { return tx.Commit() },
Rollback: func(context.Context) error { return tx.Rollback() },
} }
executed++ return writers.RunExecution(ctx, w.options, &writers.ExecutionReport{Dialect: "SQLite"}, writers.StatementOperations(collector.statements), backend)
}
w.options.Metadata["execution_total"] = total
w.options.Metadata["execution_success"] = executed
w.options.Metadata["execution_failed"] = len(execErrors)
if len(execErrors) > 0 {
return fmt.Errorf("failed to execute %d/%d statement(s) against %q:\n%s", len(execErrors), total, dbPath, strings.Join(execErrors, "\n"))
}
return nil
} }
// truncateStatement shortens a SQL statement for error messages. // truncateStatement shortens a SQL statement for error messages.
+35 -1
View File
@@ -2,6 +2,7 @@ package sqlite
import ( import (
"database/sql" "database/sql"
"errors"
"os" "os"
"path/filepath" "path/filepath"
"strings" "strings"
@@ -182,7 +183,7 @@ func TestExecute_StopsOnErrorUnlessIgnored(t *testing.T) {
path := prepare(t) path := prepare(t)
opts := &writers.WriterOptions{Metadata: map[string]any{"connection_string": path}} opts := &writers.WriterOptions{Metadata: map[string]any{"connection_string": path}}
err := NewWriter(opts).WriteDatabase(shopDB()) err := NewWriter(opts).WriteDatabase(shopDB())
if err == nil || !strings.Contains(err.Error(), "failed to execute") { if err == nil || !strings.Contains(err.Error(), "execution failed") {
t.Fatalf("expected failure, got %v", err) t.Fatalf("expected failure, got %v", err)
} }
if opts.Metadata["execution_failed"] != 1 { if opts.Metadata["execution_failed"] != 1 {
@@ -248,3 +249,36 @@ func TestDBMLFixtureExecutes(t *testing.T) {
t.Errorf("nothing executed: %+v", opts.Metadata) t.Errorf("nothing executed: %+v", opts.Metadata)
} }
} }
func TestExecuteAtomicRollback(t *testing.T) {
path := filepath.Join(t.TempDir(), "atomic.db")
opts := &writers.WriterOptions{AtomicExecution: true, Metadata: map[string]any{"connection_string": path}}
db := models.InitDatabase("atomic")
schema := models.InitSchema("main")
for _, name := range []string{"first", "bad", "last"} {
table := models.InitTable(name, "main")
col := models.InitColumn("id", name, "main")
col.Type = "integer"
table.Columns[col.Name] = col
schema.Tables = append(schema.Tables, table)
}
// A duplicate column creates an invalid table definition in the middle.
bad := schema.Tables[1]
bad.Columns["duplicate"] = models.InitColumn("id", "bad", "main")
bad.Columns["duplicate"].Type = "integer"
db.Schemas = append(db.Schemas, schema)
err := NewWriter(opts).WriteDatabase(db)
var failure *writers.ExecutionFailure
if !errors.As(err, &failure) || failure.Report.Outcome != "rolled_back" || failure.Report.RolledBackStatements == 0 || failure.Report.ExecutedStatements != 0 {
t.Fatalf("rollback report: %v", err)
}
conn, err := sql.Open("sqlite", path)
if err != nil {
t.Fatal(err)
}
defer conn.Close()
var count int
if err := conn.QueryRow("SELECT count(*) FROM sqlite_master WHERE type='table'").Scan(&count); err != nil || count != 0 {
t.Fatalf("tables survived rollback: %d %v", count, err)
}
}
+14 -7
View File
@@ -168,15 +168,22 @@ relspec templ --from json --from-path schema.json --template test.tmpl
## Multi-file Output ## Multi-file Output
For multi-file modes, the writer: For multi-file modes, `OutputPath` is always the output directory, including when
it does not yet exist or has no trailing slash. An empty path uses the current
directory. Database mode continues to treat `OutputPath` as one explicit file.
1. Iterates through items (schemas/scripts/tables) The writer renders and validates all filenames and contents before writing. Names
2. Creates `TemplateData` for each item must be relative, may contain safe subdirectories, and cannot contain traversal,
3. Executes template with item data absolute/drive/UNC paths, backslashes, Windows device names, alternate streams,
4. Generates filename using `filename_pattern` template control characters, or trailing dots/spaces. Duplicate filenames (including case
5. Writes output to generated filename variants) and file/directory collisions fail before any artifact is replaced.
Output directory is created automatically if it doesn't exist. Output directories are created automatically. Existing symlink roots, parents, and
files are rejected. Writes and renames use `os.Root` to retain containment if paths
change during generation. Each file is staged exclusively, closed successfully,
and renamed into place; existing permissions are preserved on replacement.
Multi-file publication is per file, so an I/O error partway through finalization can
still leave a mixed generation (the broader audit A16 staging/manifest work remains).
## Filename Pattern Execution ## Filename Pattern Execution
+118 -90
View File
@@ -2,6 +2,9 @@ package template
import ( import (
"bytes" "bytes"
"crypto/rand"
"encoding/hex"
"errors"
"fmt" "fmt"
"os" "os"
"path/filepath" "path/filepath"
@@ -9,6 +12,7 @@ import (
"text/template" "text/template"
"git.warky.dev/wdevs/relspecgo/pkg/models" "git.warky.dev/wdevs/relspecgo/pkg/models"
"git.warky.dev/wdevs/relspecgo/pkg/safefile"
"git.warky.dev/wdevs/relspecgo/pkg/writers" "git.warky.dev/wdevs/relspecgo/pkg/writers"
) )
@@ -125,96 +129,138 @@ func (w *Writer) executeDatabaseMode(db *models.Database) error {
return w.writeOutput(output, w.options.OutputPath) return w.writeOutput(output, w.options.OutputPath)
} }
// executeSchemaMode executes the template once per schema // Multi-file generation renders and validates every artifact before touching output.
func (w *Writer) executeSchemaMode(db *models.Database) error { func (w *Writer) executeSchemaMode(db *models.Database) error {
var data []*TemplateData
for _, schema := range db.Schemas { for _, schema := range db.Schemas {
data := NewSchemaData(schema, w.options.Metadata) data = append(data, NewSchemaData(schema, w.options.Metadata))
output, err := w.executeTemplate(data)
if err != nil {
return fmt.Errorf("failed to execute template for schema %s: %w", schema.Name, err)
} }
return w.executeFiles(data)
filename, err := w.generateFilename(data)
if err != nil {
return fmt.Errorf("failed to generate filename for schema %s: %w", schema.Name, err)
}
if err := w.writeOutput(output, filename); err != nil {
return fmt.Errorf("failed to write output for schema %s: %w", schema.Name, err)
}
}
return nil
} }
// executeDomainMode executes the template once per domain
func (w *Writer) executeDomainMode(db *models.Database) error { func (w *Writer) executeDomainMode(db *models.Database) error {
var data []*TemplateData
for _, domain := range db.Domains { for _, domain := range db.Domains {
data := NewDomainData(domain, db, w.options.Metadata) data = append(data, NewDomainData(domain, db, w.options.Metadata))
output, err := w.executeTemplate(data)
if err != nil {
return fmt.Errorf("failed to execute template for domain %s: %w", domain.Name, err)
} }
return w.executeFiles(data)
filename, err := w.generateFilename(data)
if err != nil {
return fmt.Errorf("failed to generate filename for domain %s: %w", domain.Name, err)
}
if err := w.writeOutput(output, filename); err != nil {
return fmt.Errorf("failed to write output for domain %s: %w", domain.Name, err)
}
}
return nil
} }
// executeScriptMode executes the template once per script
func (w *Writer) executeScriptMode(db *models.Database) error { func (w *Writer) executeScriptMode(db *models.Database) error {
var data []*TemplateData
for _, schema := range db.Schemas { for _, schema := range db.Schemas {
for _, script := range schema.Scripts { for _, script := range schema.Scripts {
data := NewScriptData(script, schema, db, w.options.Metadata) data = append(data, NewScriptData(script, schema, db, w.options.Metadata))
output, err := w.executeTemplate(data) }
}
return w.executeFiles(data)
}
func (w *Writer) executeTableMode(db *models.Database) error {
var data []*TemplateData
for _, schema := range db.Schemas {
for _, table := range schema.Tables {
data = append(data, NewTableData(table, schema, db, w.options.Metadata))
}
}
return w.executeFiles(data)
}
type artifact struct{ name, content string }
func (w *Writer) outputRoot() string {
if w.options.OutputPath == "" {
return "."
}
return w.options.OutputPath
}
func (w *Writer) executeFiles(data []*TemplateData) error {
var files []artifact
seen := make(map[string]bool)
for _, item := range data {
name, err := w.renderFilename(item)
if err != nil { if err != nil {
return fmt.Errorf("failed to execute template for script %s: %w", script.Name, err) return err
} }
key := strings.ToLower(filepath.ToSlash(name))
filename, err := w.generateFilename(data) if seen[key] {
return fmt.Errorf("generated filename collision: %q", name)
}
seen[key] = true
content, err := w.executeTemplate(item)
if err != nil { if err != nil {
return fmt.Errorf("failed to generate filename for script %s: %w", script.Name, err) return err
} }
files = append(files, artifact{name, content})
if err := w.writeOutput(output, filename); err != nil { }
return fmt.Errorf("failed to write output for script %s: %w", script.Name, err) // Reject file/directory conflicts before creating any output.
for name := range seen {
for parent := filepath.ToSlash(filepath.Dir(name)); parent != "."; parent = filepath.ToSlash(filepath.Dir(parent)) {
if seen[parent] {
return fmt.Errorf("generated file/directory collision: %q", name)
} }
} }
} }
if len(files) == 0 {
return nil
}
rootPath := w.outputRoot()
if err := safefile.CheckRoot(rootPath); err != nil {
return err
}
if err := os.MkdirAll(rootPath, 0o755); err != nil {
return err
}
root, err := os.OpenRoot(rootPath)
if err != nil {
return err
}
defer root.Close()
// Preflight every destination, including existing directory and symlink conflicts.
for _, file := range files {
if err := safefile.Check(root, file.name); err != nil {
return err
}
}
for _, file := range files {
if err := w.writeArtifact(root, file); err != nil {
return fmt.Errorf("write generated file %q: %w", file.name, err)
}
}
return nil return nil
} }
// executeTableMode executes the template once per table func (w *Writer) writeArtifact(root *os.Root, file artifact) error {
func (w *Writer) executeTableMode(db *models.Database) error { if err := root.MkdirAll(filepath.Dir(file.name), 0o755); err != nil {
for _, schema := range db.Schemas { return err
for _, table := range schema.Tables { }
data := NewTableData(table, schema, db, w.options.Metadata) // Exclusive unpredictable staging avoids truncating a destination (or a link target).
output, err := w.executeTemplate(data) var token [16]byte
if _, err := rand.Read(token[:]); err != nil {
return err
}
temporary := ".relspec-" + hex.EncodeToString(token[:]) + ".tmp"
mode := os.FileMode(0o644)
if info, err := root.Lstat(file.name); err == nil && info.Mode().IsRegular() {
mode = info.Mode().Perm()
}
staged, err := root.OpenFile(temporary, os.O_WRONLY|os.O_CREATE|os.O_EXCL, mode)
if err != nil { if err != nil {
return fmt.Errorf("failed to execute template for table %s.%s: %w", schema.Name, table.Name, err) return err
} }
defer func() {
filename, err := w.generateFilename(data) // Best-effort cleanup; a successful rename already removes the staging path.
if err != nil { _ = root.Remove(temporary)
return fmt.Errorf("failed to generate filename for table %s.%s: %w", schema.Name, table.Name, err) }()
_, writeErr := staged.WriteString(file.content)
if err := errors.Join(writeErr, staged.Close()); err != nil {
return err
} }
if err := safefile.Check(root, file.name); err != nil {
if err := w.writeOutput(output, filename); err != nil { return err
return fmt.Errorf("failed to write output for table %s.%s: %w", schema.Name, table.Name, err)
} }
} return root.Rename(temporary, file.name)
}
return nil
} }
// loadTemplate loads and parses the template file // loadTemplate loads and parses the template file
@@ -245,43 +291,25 @@ func (w *Writer) executeTemplate(data *TemplateData) (string, error) {
return buf.String(), nil return buf.String(), nil
} }
// generateFilename generates a filename from the filename pattern // renderFilename returns a validated path relative to the explicit output root.
func (w *Writer) generateFilename(data *TemplateData) (string, error) { func (w *Writer) renderFilename(data *TemplateData) (string, error) {
// Parse filename pattern as a template
tmpl, err := template.New("filename").Funcs(w.funcMap).Parse(w.filenamePattern) tmpl, err := template.New("filename").Funcs(w.funcMap).Parse(w.filenamePattern)
if err != nil { if err != nil {
return "", fmt.Errorf("invalid filename pattern: %w", err) return "", fmt.Errorf("invalid filename pattern: %w", err)
} }
// Execute filename template
var buf bytes.Buffer var buf bytes.Buffer
if err := tmpl.Execute(&buf, data); err != nil { if err := tmpl.Execute(&buf, data); err != nil {
return "", fmt.Errorf("failed to generate filename: %w", err) return "", fmt.Errorf("failed to generate filename: %w", err)
} }
return safefile.Relative(buf.String())
}
filename := buf.String() func (w *Writer) generateFilename(data *TemplateData) (string, error) {
name, err := w.renderFilename(data)
// If output path is a directory, join with generated filename if err != nil {
if w.options.OutputPath != "" { return "", err
// Check if output path is a directory
info, err := os.Stat(w.options.OutputPath)
if err == nil && info.IsDir() {
filename = filepath.Join(w.options.OutputPath, filename)
} else {
// If it doesn't exist, check if it looks like a directory (ends with /)
if strings.HasSuffix(w.options.OutputPath, string(filepath.Separator)) {
filename = filepath.Join(w.options.OutputPath, filename)
} else {
// Use output path as base directory
dir := filepath.Dir(w.options.OutputPath)
if dir != "." {
filename = filepath.Join(dir, filename)
} }
} return filepath.Join(w.outputRoot(), name), nil
}
}
return filename, nil
} }
// writeOutput writes the output to a file or stdout // writeOutput writes the output to a file or stdout
+122 -2
View File
@@ -180,8 +180,8 @@ func TestWriterGenerateFilenameOutputPathForms(t *testing.T) {
{"no output path", "", "users.txt"}, {"no output path", "", "users.txt"},
{"existing dir", dir, filepath.Join(dir, "users.txt")}, {"existing dir", dir, filepath.Join(dir, "users.txt")},
{"trailing separator", filepath.Join(dir, "new") + string(filepath.Separator), filepath.Join(dir, "new", "users.txt")}, {"trailing separator", filepath.Join(dir, "new") + string(filepath.Separator), filepath.Join(dir, "new", "users.txt")},
{"file path uses its dir", filepath.Join(dir, "x.out"), filepath.Join(dir, "users.txt")}, {"nonexistent path is root", filepath.Join(dir, "x.out"), filepath.Join(dir, "x.out", "users.txt")},
{"bare file name", "x.out", "users.txt"}, {"bare path is root", "x.out", filepath.Join("x.out", "users.txt")},
} }
for _, tt := range tests { for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) { t.Run(tt.name, func(t *testing.T) {
@@ -217,3 +217,123 @@ func TestWriterWriteSchemaAndTable(t *testing.T) {
t.Errorf("WriteTable: %q", data) t.Errorf("WriteTable: %q", data)
} }
} }
func TestMultiFileRejectsUnsafeFilenamesBeforeWriting(t *testing.T) {
for _, name := range []string{"../escape", "/absolute", `..\escape`, `C:\escape`, `\\server\share\file`, "a/../escape", "a//escape", "CON", "nul.txt", "COM¹.txt", "file:stream", "trailing.", "trailing ", "line\nbreak"} {
t.Run(name, func(t *testing.T) {
out := filepath.Join(t.TempDir(), "new-output")
w, err := newTestWriter(t, "content", "schema", "{{.Name}}", out)
if err != nil {
t.Fatal(err)
}
db := models.InitDatabase("unsafe")
db.Schemas = []*models.Schema{models.InitSchema("valid"), models.InitSchema(name)}
if err := w.WriteDatabase(db); err == nil {
t.Fatal("expected rejected filename")
}
if _, err := os.Stat(out); !os.IsNotExist(err) {
t.Fatalf("validation failure must not create output: %v", err)
}
})
}
}
func TestMultiFileCollisionsDoNotOverwrite(t *testing.T) {
for _, names := range [][]string{{"same.txt", "same.txt"}, {"SAME.txt", "same.txt"}, {"file", "file/child.txt"}} {
out := t.TempDir()
seed := filepath.Join(out, "same.txt")
if err := os.WriteFile(seed, []byte("original"), 0o600); err != nil {
t.Fatal(err)
}
w, err := newTestWriter(t, "replacement", "schema", "{{.Name}}", out)
if err != nil {
t.Fatal(err)
}
db := models.InitDatabase("collision")
for _, name := range names {
db.Schemas = append(db.Schemas, models.InitSchema(name))
}
if err := w.WriteDatabase(db); err == nil || !strings.Contains(err.Error(), "collision") {
t.Fatalf("expected collision error: %v", err)
}
got, err := os.ReadFile(seed)
if err != nil || string(got) != "original" {
t.Fatalf("existing output changed: %q %v", got, err)
}
}
}
func TestMultiFileRejectsSymlinksBeforeAnyWrites(t *testing.T) {
for _, kind := range []string{"parent", "leaf", "root"} {
t.Run(kind, func(t *testing.T) {
out, external := t.TempDir(), t.TempDir()
secret := filepath.Join(external, "secret.txt")
if err := os.WriteFile(secret, []byte("original"), 0o600); err != nil {
t.Fatal(err)
}
link, target := filepath.Join(out, "link"), external
pattern := "{{.Name}}/secret.txt"
if kind == "leaf" {
target = secret
pattern = "{{.Name}}"
}
if kind == "root" {
link = filepath.Join(out, "root")
out = link
pattern = "{{.Name}}"
}
if err := os.Symlink(target, link); err != nil {
t.Skip(err)
}
w, err := newTestWriter(t, "replacement", "schema", pattern, out)
if err != nil {
t.Fatal(err)
}
db := models.InitDatabase("links")
db.Schemas = []*models.Schema{models.InitSchema("first"), models.InitSchema("link")}
if err := w.WriteDatabase(db); err == nil {
t.Fatal("expected symlink rejection")
}
got, err := os.ReadFile(secret)
if err != nil || string(got) != "original" {
t.Fatalf("symlink target changed: %q %v", got, err)
}
if _, err := os.Stat(filepath.Join(out, "first")); !os.IsNotExist(err) {
t.Fatalf("preflight must precede writes: %v", err)
}
})
}
}
func TestMultiFileNewDirectoryAndNestedFiles(t *testing.T) {
out := filepath.Join(t.TempDir(), "new-output")
w, err := newTestWriter(t, "{{.Table.Name}}", "table", "{{.ParentSchema.Name}}/{{.Name}}.txt", out)
if err != nil {
t.Fatal(err)
}
if err := w.WriteDatabase(modeDB()); err != nil {
t.Fatal(err)
}
got, err := os.ReadFile(filepath.Join(out, "a", "t1.txt"))
if err != nil || string(got) != "t1" {
t.Fatalf("nested output: %q %v", got, err)
}
}
func TestMultiFileRootAncestorSymlinkRejected(t *testing.T) {
parent, external := t.TempDir(), t.TempDir()
link := filepath.Join(parent, "link")
if err := os.Symlink(external, link); err != nil {
t.Skip(err)
}
w, err := newTestWriter(t, "content", "schema", "{{.Name}}.txt", filepath.Join(link, "new"))
if err != nil {
t.Fatal(err)
}
if err := w.WriteDatabase(modeDB()); err == nil {
t.Fatal("root ancestor symlink must fail")
}
if _, err := os.Stat(filepath.Join(external, "new")); !os.IsNotExist(err) {
t.Fatalf("must not create external directories: %v", err)
}
}

Some files were not shown because too many files have changed in this diff Show More