Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
32cf25ff73 | ||
|
|
032093e8c0 | ||
|
|
69a27784d1 | ||
|
|
1d93255575 | ||
|
|
05466a92b1 | ||
|
|
dd97413b8e | ||
|
|
2fa48335d3 | ||
|
|
bfe9852826 |
@@ -71,11 +71,18 @@ jobs:
|
|||||||
EXT=""
|
EXT=""
|
||||||
[ "$GOOS" = "windows" ] && EXT=".exe"
|
[ "$GOOS" = "windows" ] && EXT=".exe"
|
||||||
NAME="relspec-${GOOS}-${GOARCH}${EXT}"
|
NAME="relspec-${GOOS}-${GOARCH}${EXT}"
|
||||||
GOOS="$GOOS" GOARCH="$GOARCH" go build \
|
GOOS="$GOOS" GOARCH="$GOARCH" CGO_ENABLED=0 go build \
|
||||||
-trimpath \
|
-trimpath \
|
||||||
-ldflags "$LDFLAGS" \
|
-ldflags "$LDFLAGS" \
|
||||||
-o "$NAME" ./cmd/relspec
|
-o "$NAME" ./cmd/relspec
|
||||||
echo "Built $NAME"
|
echo "Built $NAME"
|
||||||
|
if [ "$GOOS" = "linux" ]; then
|
||||||
|
TARNAME="relspec_${VERSION#v}_${GOOS}_${GOARCH}.tar.gz"
|
||||||
|
cp "$NAME" relspec
|
||||||
|
tar czf "$TARNAME" relspec LICENSE
|
||||||
|
rm relspec
|
||||||
|
echo "Packed $TARNAME"
|
||||||
|
fi
|
||||||
done
|
done
|
||||||
|
|
||||||
- name: Build Windows installer
|
- name: Build Windows installer
|
||||||
@@ -121,7 +128,7 @@ jobs:
|
|||||||
exit 1
|
exit 1
|
||||||
fi
|
fi
|
||||||
|
|
||||||
for f in relspec-*; do
|
for f in relspec-* relspec_*.tar.gz; do
|
||||||
echo "Uploading $f..."
|
echo "Uploading $f..."
|
||||||
curl -s -X POST "${UPLOAD_URL}?name=${f}" \
|
curl -s -X POST "${UPLOAD_URL}?name=${f}" \
|
||||||
-H "Authorization: token ${GITHUB_TOKEN}" \
|
-H "Authorization: token ${GITHUB_TOKEN}" \
|
||||||
@@ -132,8 +139,13 @@ jobs:
|
|||||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||||
|
|
||||||
pkg-aur:
|
pkg-aur:
|
||||||
|
name: AUR package (${{ matrix.pkg }})
|
||||||
needs: release
|
needs: release
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
|
strategy:
|
||||||
|
fail-fast: false
|
||||||
|
matrix:
|
||||||
|
pkg: [relspec, relspec-bin]
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@v4
|
- uses: actions/checkout@v4
|
||||||
|
|
||||||
@@ -145,6 +157,7 @@ jobs:
|
|||||||
|
|
||||||
VERSION="${{ github.event.inputs.tag || github.ref_name }}"
|
VERSION="${{ github.event.inputs.tag || github.ref_name }}"
|
||||||
PKGVER="${VERSION#v}"
|
PKGVER="${VERSION#v}"
|
||||||
|
PKG="${{ matrix.pkg }}"
|
||||||
AUR_KEY_PATH="$HOME/.ssh/aur"
|
AUR_KEY_PATH="$HOME/.ssh/aur"
|
||||||
AUR_KNOWN_HOSTS="$HOME/.ssh/known_hosts"
|
AUR_KNOWN_HOSTS="$HOME/.ssh/known_hosts"
|
||||||
|
|
||||||
@@ -183,10 +196,15 @@ jobs:
|
|||||||
|
|
||||||
# Clone AUR repo
|
# Clone AUR repo
|
||||||
GIT_SSH_COMMAND="ssh -o IdentitiesOnly=yes -o StrictHostKeyChecking=yes -o UserKnownHostsFile=$AUR_KNOWN_HOSTS -i $AUR_KEY_PATH" \
|
GIT_SSH_COMMAND="ssh -o IdentitiesOnly=yes -o StrictHostKeyChecking=yes -o UserKnownHostsFile=$AUR_KNOWN_HOSTS -i $AUR_KEY_PATH" \
|
||||||
git clone ssh://aur@aur.archlinux.org/relspec.git aur-repo
|
git clone "ssh://aur@aur.archlinux.org/${PKG}.git" aur-repo
|
||||||
|
|
||||||
|
if [ -f aur-repo/PKGBUILD ]; then
|
||||||
CURRENT_PKGVER=$(awk -F= '/^pkgver=/ {print $2; exit}' aur-repo/PKGBUILD | tr -d "[:space:]")
|
CURRENT_PKGVER=$(awk -F= '/^pkgver=/ {print $2; exit}' aur-repo/PKGBUILD | tr -d "[:space:]")
|
||||||
CURRENT_PKGREL=$(awk -F= '/^pkgrel=/ {print $2; exit}' aur-repo/PKGBUILD | tr -d "[:space:]")
|
CURRENT_PKGREL=$(awk -F= '/^pkgrel=/ {print $2; exit}' aur-repo/PKGBUILD | tr -d "[:space:]")
|
||||||
|
else
|
||||||
|
CURRENT_PKGVER=""
|
||||||
|
CURRENT_PKGREL=0
|
||||||
|
fi
|
||||||
|
|
||||||
if [ "$CURRENT_PKGVER" = "$PKGVER" ]; then
|
if [ "$CURRENT_PKGVER" = "$PKGVER" ]; then
|
||||||
case "$CURRENT_PKGREL" in
|
case "$CURRENT_PKGREL" in
|
||||||
@@ -202,16 +220,25 @@ jobs:
|
|||||||
PKGREL=1
|
PKGREL=1
|
||||||
fi
|
fi
|
||||||
|
|
||||||
echo "Publishing AUR package version ${PKGVER}-${PKGREL}"
|
echo "Publishing AUR package ${PKG} ${PKGVER}-${PKGREL}"
|
||||||
|
|
||||||
# Compute SHA256 of the source archive from the same URL the PKGBUILD will download.
|
# Compute SHA256 of the artifacts from the same URLs the PKGBUILD will download.
|
||||||
SHA=$(curl -fsSL "https://git.warky.dev/wdevs/relspecgo/releases/download/v${PKGVER}/relspec-v${PKGVER}-source.zip" | sha256sum | cut -d' ' -f1)
|
DL="https://git.warky.dev/wdevs/relspecgo/releases/download/v${PKGVER}"
|
||||||
|
if [ "$PKG" = "relspec-bin" ]; then
|
||||||
|
SHA_AMD64=$(curl -fsSL "${DL}/relspec_${PKGVER}_linux_amd64.tar.gz" | sha256sum | cut -d' ' -f1)
|
||||||
|
SHA_ARM64=$(curl -fsSL "${DL}/relspec_${PKGVER}_linux_arm64.tar.gz" | sha256sum | cut -d' ' -f1)
|
||||||
|
SUMS=(-e "s/^sha256sums_x86_64=.*/sha256sums_x86_64=('${SHA_AMD64}')/" \
|
||||||
|
-e "s/^sha256sums_aarch64=.*/sha256sums_aarch64=('${SHA_ARM64}')/")
|
||||||
|
else
|
||||||
|
SHA=$(curl -fsSL "${DL}/relspec-v${PKGVER}-source.zip" | sha256sum | cut -d' ' -f1)
|
||||||
|
SUMS=(-e "s/^sha256sums=.*/sha256sums=('${SHA}')/")
|
||||||
|
fi
|
||||||
|
|
||||||
# Update PKGBUILD — keep remote source URL, bump version/checksum, and increment pkgrel for same-version rebuilds.
|
# Update PKGBUILD — keep remote source URL, bump version/checksum, and increment pkgrel for same-version rebuilds.
|
||||||
sed -e "s/^pkgver=.*/pkgver=${PKGVER}/" \
|
sed -e "s/^pkgver=.*/pkgver=${PKGVER}/" \
|
||||||
-e "s/^pkgrel=.*/pkgrel=${PKGREL}/" \
|
-e "s/^pkgrel=.*/pkgrel=${PKGREL}/" \
|
||||||
-e "s/^sha256sums=.*/sha256sums=('${SHA}')/" \
|
"${SUMS[@]}" \
|
||||||
linux/arch/PKGBUILD > aur-repo/PKGBUILD
|
"linux/arch/${PKG}/PKGBUILD" > aur-repo/PKGBUILD
|
||||||
|
|
||||||
# Generate .SRCINFO inside an Arch container (docker cp avoids DinD volume mount issues)
|
# Generate .SRCINFO inside an Arch container (docker cp avoids DinD volume mount issues)
|
||||||
CID=$(docker run -d archlinux:latest sleep infinity)
|
CID=$(docker run -d archlinux:latest sleep infinity)
|
||||||
|
|||||||
@@ -17,7 +17,8 @@ GOCLEAN=$(GOCMD) clean
|
|||||||
# Tool versions (compiled on demand via `go run` so they match the local toolchain)
|
# Tool versions (compiled on demand via `go run` so they match the local toolchain)
|
||||||
GOLANGCI_LINT = go run github.com/golangci/golangci-lint/v2/cmd/golangci-lint@latest
|
GOLANGCI_LINT = go run github.com/golangci/golangci-lint/v2/cmd/golangci-lint@latest
|
||||||
STATICCHECK = build/tools/staticcheck
|
STATICCHECK = build/tools/staticcheck
|
||||||
GOVULNCHECK = go run golang.org/x/vuln/cmd/govulncheck@latest
|
# Scan with the toolchain pinned in go.mod (what CI uses), so stdlib vulns match CI.
|
||||||
|
GOVULNCHECK = GOTOOLCHAIN=go$(shell awk '/^go /{print $$2; exit}' go.mod) go run golang.org/x/vuln/cmd/govulncheck@latest
|
||||||
|
|
||||||
# Version information
|
# Version information
|
||||||
# VERSION file is the single source of truth (bumped by `make release-version`).
|
# VERSION file is the single source of truth (bumped by `make release-version`).
|
||||||
@@ -218,7 +219,7 @@ docker-test-integration: docker-up ## Start DB and run integration tests
|
|||||||
$(GOTEST) -v ./pkg/readers/pgsql/ -count=1 || (make docker-down && exit 1)
|
$(GOTEST) -v ./pkg/readers/pgsql/ -count=1 || (make docker-down && exit 1)
|
||||||
@make docker-down
|
@make docker-down
|
||||||
|
|
||||||
release: lint fmt-check test build ## Run lint, format check, tests, build, then create and push a new release tag
|
release: lint fmt-check govulncheck test build ## Run lint, format check, govulncheck, tests, build, then create and push a new release tag
|
||||||
@echo "Creating new release..."
|
@echo "Creating new release..."
|
||||||
@latest_tag=$$(git describe --tags --abbrev=0 2>/dev/null || echo ""); \
|
@latest_tag=$$(git describe --tags --abbrev=0 2>/dev/null || echo ""); \
|
||||||
if [ -z "$$latest_tag" ]; then \
|
if [ -z "$$latest_tag" ]; then \
|
||||||
@@ -237,9 +238,9 @@ release: lint fmt-check test build ## Run lint, format check, tests, build, then
|
|||||||
PKGVER=$${version#v}; \
|
PKGVER=$${version#v}; \
|
||||||
echo "Updating package metadata to $$PKGVER..."; \
|
echo "Updating package metadata to $$PKGVER..."; \
|
||||||
echo "$$PKGVER" > VERSION; \
|
echo "$$PKGVER" > VERSION; \
|
||||||
sed -i "s/^pkgver=.*/pkgver=$$PKGVER/" linux/arch/PKGBUILD; \
|
sed -i "s/^pkgver=.*/pkgver=$$PKGVER/" linux/arch/relspec/PKGBUILD linux/arch/relspec-bin/PKGBUILD; \
|
||||||
sed -i "s/^Version:.*/Version: $$PKGVER/" linux/centos/relspec.spec; \
|
sed -i "s/^Version:.*/Version: $$PKGVER/" linux/centos/relspec.spec; \
|
||||||
git add VERSION linux/arch/PKGBUILD linux/centos/relspec.spec; \
|
git add VERSION linux/arch/relspec/PKGBUILD linux/arch/relspec-bin/PKGBUILD linux/centos/relspec.spec; \
|
||||||
if ! git diff --cached --quiet; then \
|
if ! git diff --cached --quiet; then \
|
||||||
git commit -m "chore(release): update package version to $$PKGVER"; \
|
git commit -m "chore(release): update package version to $$PKGVER"; \
|
||||||
fi; \
|
fi; \
|
||||||
@@ -252,7 +253,7 @@ release: lint fmt-check test build ## Run lint, format check, tests, build, then
|
|||||||
git push origin "$$version"; \
|
git push origin "$$version"; \
|
||||||
echo "Tag $$version created and pushed to remote repository."
|
echo "Tag $$version created and pushed to remote repository."
|
||||||
|
|
||||||
release-version: lint fmt-check staticcheck ## Run lint, format check and staticcheck, then auto-increment patch version, update VERSION/package files, commit, tag, and push
|
release-version: lint fmt-check staticcheck govulncheck ## Run lint, format check, staticcheck and govulncheck, then auto-increment patch version, update VERSION/package files, commit, tag, and push
|
||||||
@CURRENT=$$([ -f VERSION ] && cat VERSION | tr -d '[:space:]' || (git describe --tags --abbrev=0 2>/dev/null | sed 's/^v//') || echo "0.0.0"); \
|
@CURRENT=$$([ -f VERSION ] && cat VERSION | tr -d '[:space:]' || (git describe --tags --abbrev=0 2>/dev/null | sed 's/^v//') || echo "0.0.0"); \
|
||||||
MAJOR=$$(echo $$CURRENT | cut -d. -f1); \
|
MAJOR=$$(echo $$CURRENT | cut -d. -f1); \
|
||||||
MINOR=$$(echo $$CURRENT | cut -d. -f2); \
|
MINOR=$$(echo $$CURRENT | cut -d. -f2); \
|
||||||
@@ -261,15 +262,15 @@ release-version: lint fmt-check staticcheck ## Run lint, format check and static
|
|||||||
NEXT="v$$PKGVER"; \
|
NEXT="v$$PKGVER"; \
|
||||||
echo "Current: v$$CURRENT → Next: $$NEXT"; \
|
echo "Current: v$$CURRENT → Next: $$NEXT"; \
|
||||||
echo "$$PKGVER" > VERSION; \
|
echo "$$PKGVER" > VERSION; \
|
||||||
sed -i "s/^pkgver=.*/pkgver=$$PKGVER/" linux/arch/PKGBUILD; \
|
sed -i "s/^pkgver=.*/pkgver=$$PKGVER/" linux/arch/relspec/PKGBUILD linux/arch/relspec-bin/PKGBUILD; \
|
||||||
sed -i "s/^Version:.*/Version: $$PKGVER/" linux/centos/relspec.spec; \
|
sed -i "s/^Version:.*/Version: $$PKGVER/" linux/centos/relspec.spec; \
|
||||||
git add VERSION linux/arch/PKGBUILD linux/centos/relspec.spec; \
|
git add VERSION linux/arch/relspec/PKGBUILD linux/arch/relspec-bin/PKGBUILD linux/centos/relspec.spec; \
|
||||||
git commit -m "chore(release): update package version to $$PKGVER"; \
|
git commit -m "chore(release): update package version to $$PKGVER"; \
|
||||||
git tag -a "$$NEXT" -m "Release $$NEXT"; \
|
git tag -a "$$NEXT" -m "Release $$NEXT"; \
|
||||||
git push origin HEAD "$$NEXT"; \
|
git push origin HEAD "$$NEXT"; \
|
||||||
echo "Pushed $$NEXT — release workflow triggered"
|
echo "Pushed $$NEXT — release workflow triggered"
|
||||||
|
|
||||||
rerelease: lint fmt-check ## Move the latest tag to HEAD and force push it
|
rerelease: lint fmt-check govulncheck ## Move the latest tag to HEAD and force push it
|
||||||
@TAG=$$(git describe --tags --abbrev=0 2>/dev/null); \
|
@TAG=$$(git describe --tags --abbrev=0 2>/dev/null); \
|
||||||
if [ -z "$$TAG" ]; then echo "No existing tags found"; exit 1; fi; \
|
if [ -z "$$TAG" ]; then echo "No existing tags found"; exit 1; fi; \
|
||||||
echo "Moving $$TAG to $$(git rev-parse --short HEAD)"; \
|
echo "Moving $$TAG to $$(git rev-parse --short HEAD)"; \
|
||||||
|
|||||||
@@ -41,6 +41,9 @@ relspec convert --from pgsql --from-conn "postgres://user:pass@localhost/mydb" \
|
|||||||
# DBML → PostgreSQL DDL
|
# DBML → PostgreSQL DDL
|
||||||
relspec convert --from dbml --from-path schema.dbml --to pgsql --to-path schema.sql
|
relspec convert --from dbml --from-path schema.dbml --to pgsql --to-path schema.sql
|
||||||
|
|
||||||
|
# Lowercase model names and structured references before output
|
||||||
|
relspec convert --from dbml --from-path schema.dbml --to pgsql --to-path schema.sql --lowercase
|
||||||
|
|
||||||
# PostgreSQL → SQLite (auto flattens schemas)
|
# PostgreSQL → SQLite (auto flattens schemas)
|
||||||
relspec convert --from pgsql --from-conn "postgres://..." --to sqlite --to-path schema.sql
|
relspec convert --from pgsql --from-conn "postgres://..." --to sqlite --to-path schema.sql
|
||||||
|
|
||||||
@@ -57,6 +60,44 @@ connections or `--dry-run`). Source files are polled every `--watch-interval`
|
|||||||
is ignored so generating into the source tree does not loop. Conversion errors
|
is ignored so generating into the source tree does not loop. Conversion errors
|
||||||
are printed and watching continues.
|
are printed and watching continues.
|
||||||
|
|
||||||
|
#### Identifier casing and PostgreSQL quoting
|
||||||
|
|
||||||
|
PostgreSQL output always double-quotes model identifiers, including reserved
|
||||||
|
words, and preserves their case by default. For example, schema `Sales` and table
|
||||||
|
`Order` produce `"Sales"."Order"`; with `--lowercase`, they produce
|
||||||
|
`"sales"."order"`. Quoted names are case-sensitive in PostgreSQL.
|
||||||
|
|
||||||
|
`--lowercase` is available on `convert`, `batch`, `merge`, `split`, `templ`, and
|
||||||
|
`job run`. It lowercases model names and structured references on a copy and
|
||||||
|
rejects collisions such as `Users` and `users` in the same schema. SQL expressions,
|
||||||
|
defaults, comments, enum values, and data retain their case. Adjust identifier
|
||||||
|
references inside your own SQL expressions when opting into lowercasing.
|
||||||
|
|
||||||
|
Generated PostgreSQL SQL also escapes identifier quotes and literal values and
|
||||||
|
chooses dollar-quote delimiters that do not collide with function or procedural
|
||||||
|
block contents. Raw SQL expressions remain executable input. See the
|
||||||
|
[PostgreSQL writer documentation](pkg/writers/pgsql/README.md#identifier-and-value-handling)
|
||||||
|
for library options and details.
|
||||||
|
|
||||||
|
#### MySQL fidelity and strict conversion
|
||||||
|
|
||||||
|
MySQL output now preserves foreign keys and their actions, checks, indexes,
|
||||||
|
unsigned types, dimensions, generated columns, and native defaults. Known
|
||||||
|
unsupported features produce warnings. Add `--strict-conversion` to reject
|
||||||
|
lossy MySQL output before writing a file or executing DDL:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
relspec convert --from mysql --from-conn "$MYSQL_DSN" \
|
||||||
|
--to mysql --to-path schema.sql --strict-conversion
|
||||||
|
```
|
||||||
|
|
||||||
|
Schema jobs can set `options.strict_conversion: true`. This check currently
|
||||||
|
covers MySQL output; it does not guarantee lossless conversion to every format.
|
||||||
|
The live suite covers MySQL 8.4 / InnoDB, including JSON/YAML intermediates.
|
||||||
|
See the [MySQL capability matrix](docs/MYSQL_FIDELITY.md) for unsupported objects,
|
||||||
|
metadata, SQL-mode requirements, and execution limits. Output creates tables in
|
||||||
|
existing databases; it is not a data copy or a rerunnable migration.
|
||||||
|
|
||||||
### `batch` — Convert many inputs in one run
|
### `batch` — Convert many inputs in one run
|
||||||
|
|
||||||
Converts each input independently (one output per input, unlike `--from-list`
|
Converts each input independently (one output per input, unlike `--from-list`
|
||||||
@@ -165,6 +206,7 @@ jobs:
|
|||||||
overwrite: true
|
overwrite: true
|
||||||
options:
|
options:
|
||||||
flatten_schema: false
|
flatten_schema: false
|
||||||
|
lowercase_identifiers: true # opt in; defaults to false
|
||||||
logfile: .relspec/log/build-schema.log
|
logfile: .relspec/log/build-schema.log
|
||||||
```
|
```
|
||||||
|
|
||||||
@@ -177,6 +219,40 @@ any job runs. Path-like fields also support `${NAME}` environment-variable
|
|||||||
references, which are expanded and safety-checked during pre-flight. See
|
references, which are expanded and safety-checked during pre-flight. See
|
||||||
[docs/JOB_FILES.md](docs/JOB_FILES.md).
|
[docs/JOB_FILES.md](docs/JOB_FILES.md).
|
||||||
|
|
||||||
|
Use `options.lowercase_identifiers: true` per job, or
|
||||||
|
`relspec job run build-schema --lowercase` to enable lowercasing for schema outputs
|
||||||
|
throughout the selected job plan. For MySQL schema output,
|
||||||
|
`options.strict_conversion: true` or `--strict-conversion` rejects known lossy
|
||||||
|
conversions before writing.
|
||||||
|
|
||||||
|
### `scripts` — SQL scripts and embedded assets
|
||||||
|
|
||||||
|
```bash
|
||||||
|
relspec scripts list --dir ./migrations
|
||||||
|
relspec scripts execute --dir ./migrations --conn "$DATABASE_URL"
|
||||||
|
```
|
||||||
|
|
||||||
|
Execution stops on the first error by default. `--continue-on-error` (alias
|
||||||
|
`--coe`) attempts later scripts and still exits nonzero if any script fails.
|
||||||
|
|
||||||
|
Scripts can embed nearby files using a standalone directive comment:
|
||||||
|
|
||||||
|
```sql
|
||||||
|
-- @embed: path=assets/message.txt var=:message mode=text
|
||||||
|
INSERT INTO messages (body) VALUES (:message);
|
||||||
|
```
|
||||||
|
|
||||||
|
Embed replacement preserves dollar text such as `$1`, `${2}`, and `$HOME` exactly.
|
||||||
|
Placeholders are replaced only in executable SQL; quoted strings, quoted
|
||||||
|
identifiers, comments, casts, and dollar-quoted bodies retain their contents.
|
||||||
|
Inserted content is never scanned for further placeholders. Asset-call named
|
||||||
|
parameters follow the same SQL lexical boundaries. See
|
||||||
|
[docs/SCRIPTS_COMMAND.md](docs/SCRIPTS_COMMAND.md) for file containment, limits,
|
||||||
|
binary embedding, and execution details.
|
||||||
|
|
||||||
|
MySQL DDL execution keeps each generated statement whole, preserving multiline
|
||||||
|
defaults containing semicolons or comment-like text.
|
||||||
|
|
||||||
### `edit` — Interactive TUI editor
|
### `edit` — Interactive TUI editor
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
|
|||||||
@@ -203,7 +203,8 @@ func runAssetsExecute(cmd *cobra.Command, args []string) error {
|
|||||||
if len(failures) > 0 {
|
if len(failures) > 0 {
|
||||||
fmt.Fprintf(os.Stderr, "Failed: %d\n", len(failures))
|
fmt.Fprintf(os.Stderr, "Failed: %d\n", len(failures))
|
||||||
fmt.Fprintf(os.Stderr, "\n⚠ Failed Entries Summary (%d failed):\n", len(failures))
|
fmt.Fprintf(os.Stderr, "\n⚠ Failed Entries Summary (%d failed):\n", len(failures))
|
||||||
for i, f := range failures {
|
for i := range failures {
|
||||||
|
f := &failures[i]
|
||||||
fmt.Fprintf(os.Stderr, " %d. %s (Priority=%d, Sequence=%d)\n Error: %v\n",
|
fmt.Fprintf(os.Stderr, " %d. %s (Priority=%d, Sequence=%d)\n Error: %v\n",
|
||||||
i+1, filepath.Base(f.item.Entry.File), f.item.Priority, f.item.Sequence, f.err)
|
i+1, filepath.Base(f.item.Entry.File), f.item.Priority, f.item.Sequence, f.err)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -68,6 +68,7 @@ func init() {
|
|||||||
batchCmd.Flags().StringVar(&batchNullableTypes, "types", "", "Nullable type package for code-gen writers (bun/gorm)")
|
batchCmd.Flags().StringVar(&batchNullableTypes, "types", "", "Nullable type package for code-gen writers (bun/gorm)")
|
||||||
batchCmd.Flags().StringVar(&batchNullableArrays, "array-nullable", "", "Nullable array representation for the Bun writer")
|
batchCmd.Flags().StringVar(&batchNullableArrays, "array-nullable", "", "Nullable array representation for the Bun writer")
|
||||||
batchCmd.Flags().BoolVar(&batchContinueOnError, "continue-on-error", false, "Prepend \\set ON_ERROR_STOP off to generated SQL (pgsql output only)")
|
batchCmd.Flags().BoolVar(&batchContinueOnError, "continue-on-error", false, "Prepend \\set ON_ERROR_STOP off to generated SQL (pgsql output only)")
|
||||||
|
batchCmd.Flags().BoolVar(&batchContinueOnError, "coe", false, "Alias for --continue-on-error")
|
||||||
batchCmd.Flags().BoolVar(&batchKeepGoing, "keep-going", false, "Process remaining inputs after a failure; still exits non-zero")
|
batchCmd.Flags().BoolVar(&batchKeepGoing, "keep-going", false, "Process remaining inputs after a failure; still exits non-zero")
|
||||||
batchCmd.Flags().BoolVar(&batchDryRun, "dry-run", false, "Read and validate every input and print the plan without writing any output")
|
batchCmd.Flags().BoolVar(&batchDryRun, "dry-run", false, "Read and validate every input and print the plan without writing any output")
|
||||||
|
|
||||||
|
|||||||
+31
-4
@@ -28,6 +28,7 @@ import (
|
|||||||
"git.warky.dev/wdevs/relspecgo/pkg/readers/sqlite"
|
"git.warky.dev/wdevs/relspecgo/pkg/readers/sqlite"
|
||||||
"git.warky.dev/wdevs/relspecgo/pkg/readers/typeorm"
|
"git.warky.dev/wdevs/relspecgo/pkg/readers/typeorm"
|
||||||
"git.warky.dev/wdevs/relspecgo/pkg/readers/yaml"
|
"git.warky.dev/wdevs/relspecgo/pkg/readers/yaml"
|
||||||
|
"git.warky.dev/wdevs/relspecgo/pkg/transform"
|
||||||
"git.warky.dev/wdevs/relspecgo/pkg/writers"
|
"git.warky.dev/wdevs/relspecgo/pkg/writers"
|
||||||
wbun "git.warky.dev/wdevs/relspecgo/pkg/writers/bun"
|
wbun "git.warky.dev/wdevs/relspecgo/pkg/writers/bun"
|
||||||
wdbml "git.warky.dev/wdevs/relspecgo/pkg/writers/dbml"
|
wdbml "git.warky.dev/wdevs/relspecgo/pkg/writers/dbml"
|
||||||
@@ -180,12 +181,12 @@ Examples:
|
|||||||
}
|
}
|
||||||
|
|
||||||
func init() {
|
func init() {
|
||||||
convertCmd.Flags().StringVar(&convertSourceType, "from", "", "Source format (dbml, dctx, drawdb, graphql, json, yaml, gorm, bun, drizzle, prisma, typeorm, pgsql, sqlite)")
|
convertCmd.Flags().StringVar(&convertSourceType, "from", "", "Source format (dbml, dctx, drawdb, graphql, json, yaml, gorm, bun, drizzle, prisma, typeorm, pgsql, mssql, mysql, sqlite)")
|
||||||
convertCmd.Flags().StringVar(&convertSourcePath, "from-path", "", "Source file path (for file-based formats)")
|
convertCmd.Flags().StringVar(&convertSourcePath, "from-path", "", "Source file path (for file-based formats)")
|
||||||
convertCmd.Flags().StringVar(&convertSourceConn, "from-conn", "", "Source connection string (for pgsql) or file path (for sqlite)")
|
convertCmd.Flags().StringVar(&convertSourceConn, "from-conn", "", "Source connection string (pgsql, mssql, mysql) or file path (sqlite)")
|
||||||
convertCmd.Flags().StringSliceVar(&convertFromList, "from-list", nil, "Comma-separated list of source file paths to read and merge (mutually exclusive with --from-path)")
|
convertCmd.Flags().StringSliceVar(&convertFromList, "from-list", nil, "Comma-separated list of source file paths to read and merge (mutually exclusive with --from-path)")
|
||||||
|
|
||||||
convertCmd.Flags().StringVar(&convertTargetType, "to", "", "Target format (dbml, dctx, drawdb, graphql, json, yaml, gorm, bun, drizzle, prisma, typeorm, pgsql)")
|
convertCmd.Flags().StringVar(&convertTargetType, "to", "", "Target format (dbml, dctx, drawdb, graphql, json, yaml, gorm, bun, drizzle, prisma, typeorm, pgsql, mssql, mysql, sqlite)")
|
||||||
convertCmd.Flags().StringVar(&convertTargetPath, "to-path", "", "Target output path (file or directory)")
|
convertCmd.Flags().StringVar(&convertTargetPath, "to-path", "", "Target output path (file or directory)")
|
||||||
convertCmd.Flags().StringVar(&convertPackageName, "package", "", "Package name (for code generation formats like gorm/bun)")
|
convertCmd.Flags().StringVar(&convertPackageName, "package", "", "Package name (for code generation formats like gorm/bun)")
|
||||||
convertCmd.Flags().StringVar(&convertSchemaFilter, "schema", "", "Filter to a specific schema by name (required for formats like dctx that only support single schemas)")
|
convertCmd.Flags().StringVar(&convertSchemaFilter, "schema", "", "Filter to a specific schema by name (required for formats like dctx that only support single schemas)")
|
||||||
@@ -193,6 +194,7 @@ func init() {
|
|||||||
convertCmd.Flags().StringVar(&convertNullableTypes, "types", "", "Nullable type package for code-gen writers (bun/gorm): 'baselib' (default, Go pointer types), 'stdlib' (database/sql), or 'sqltypes'")
|
convertCmd.Flags().StringVar(&convertNullableTypes, "types", "", "Nullable type package for code-gen writers (bun/gorm): 'baselib' (default, Go pointer types), 'stdlib' (database/sql), or 'sqltypes'")
|
||||||
convertCmd.Flags().StringVar(&convertNullableArrays, "array-nullable", "", "Nullable PostgreSQL array representation for the Bun writer in stdlib/baselib --types mode: 'slice' (default, plain slice) or 'pointer_slice' (*[]T, distinguishes NULL from '{}')")
|
convertCmd.Flags().StringVar(&convertNullableArrays, "array-nullable", "", "Nullable PostgreSQL array representation for the Bun writer in stdlib/baselib --types mode: 'slice' (default, plain slice) or 'pointer_slice' (*[]T, distinguishes NULL from '{}')")
|
||||||
convertCmd.Flags().BoolVar(&convertContinueOnError, "continue-on-error", false, "Prepend \\set ON_ERROR_STOP off to generated SQL so psql continues past errors (pgsql output only)")
|
convertCmd.Flags().BoolVar(&convertContinueOnError, "continue-on-error", false, "Prepend \\set ON_ERROR_STOP off to generated SQL so psql continues past errors (pgsql output only)")
|
||||||
|
convertCmd.Flags().BoolVar(&convertContinueOnError, "coe", false, "Alias for --continue-on-error")
|
||||||
convertCmd.Flags().StringVar(&convertExtraFields, "extra-fields", "", "Path to JSON file containing extra Bun model fields to inject (bun output only); fields support target_table, name, type, bun_tag, json_tag, comment")
|
convertCmd.Flags().StringVar(&convertExtraFields, "extra-fields", "", "Path to JSON file containing extra Bun model fields to inject (bun output only); fields support target_table, name, type, bun_tag, json_tag, comment")
|
||||||
|
|
||||||
convertCmd.Flags().BoolVar(&convertDryRun, "dry-run", false, "Read and validate the input and print the plan without writing any output")
|
convertCmd.Flags().BoolVar(&convertDryRun, "dry-run", false, "Read and validate the input and print the plan without writing any output")
|
||||||
@@ -447,10 +449,22 @@ func readDatabaseForConvert(dbType, filePath, connString string) (*models.Databa
|
|||||||
return db, nil
|
return db, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func writeDatabase(db *models.Database, dbType, outputPath, packageName, schemaFilter string, flattenSchema bool, nullableTypes, nullableArrays string, continueOnError bool, extraFields string) error {
|
func writeDatabase(db *models.Database, dbType, outputPath, packageName, schemaFilter string, flattenSchema bool, nullableTypes, nullableArrays string, continueOnError bool, extraFields string, strict ...bool) error {
|
||||||
|
if lowercaseIdentifiers {
|
||||||
|
var err error
|
||||||
|
schemaFilter = strings.ToLower(schemaFilter)
|
||||||
|
db, err = transform.LowercaseIdentifiers(db)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
var writer writers.Writer
|
var writer writers.Writer
|
||||||
|
|
||||||
writerOpts := newWriterOptions(outputPath, packageName, flattenSchema, nullableTypes, nullableArrays, continueOnError)
|
writerOpts := newWriterOptions(outputPath, packageName, flattenSchema, nullableTypes, nullableArrays, continueOnError)
|
||||||
|
if len(strict) > 0 {
|
||||||
|
writerOpts.StrictConversion = writerOpts.StrictConversion || strict[0]
|
||||||
|
}
|
||||||
if extraFields != "" {
|
if extraFields != "" {
|
||||||
extraFieldsJSON, err := loadExtraFields(dbType, extraFields)
|
extraFieldsJSON, err := loadExtraFields(dbType, extraFields)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -600,6 +614,19 @@ func validateWriteTarget(db *models.Database, dbType, packageName, schemaFilter,
|
|||||||
"pgsql", "postgres", "postgresql", "sql",
|
"pgsql", "postgres", "postgresql", "sql",
|
||||||
"mssql", "sqlserver", "mssql2016", "mssql2017", "mssql2019", "mssql2022",
|
"mssql", "sqlserver", "mssql2016", "mssql2017", "mssql2019", "mssql2022",
|
||||||
"sqlite", "sqlite3", "prisma", "typeorm", "graphql", "gql":
|
"sqlite", "sqlite3", "prisma", "typeorm", "graphql", "gql":
|
||||||
|
case "mysql", "mariadb":
|
||||||
|
selected := db
|
||||||
|
if schemaFilter != "" {
|
||||||
|
selected = &models.Database{}
|
||||||
|
for _, s := range db.Schemas {
|
||||||
|
if s.Name == schemaFilter {
|
||||||
|
selected.Schemas = append(selected.Schemas, s)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if _, err := wmysql.NewWriter(newWriterOptions("", packageName, false, "", "", false)).Statements(selected); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
case "gorm", "bun":
|
case "gorm", "bun":
|
||||||
if packageName == "" {
|
if packageName == "" {
|
||||||
return fmt.Errorf("package name is required for %s format (use --package flag)", dbType)
|
return fmt.Errorf("package name is required for %s format (use --package flag)", dbType)
|
||||||
|
|||||||
+22
-2
@@ -19,6 +19,7 @@ import (
|
|||||||
"git.warky.dev/wdevs/relspecgo/pkg/models"
|
"git.warky.dev/wdevs/relspecgo/pkg/models"
|
||||||
"git.warky.dev/wdevs/relspecgo/pkg/readers"
|
"git.warky.dev/wdevs/relspecgo/pkg/readers"
|
||||||
"git.warky.dev/wdevs/relspecgo/pkg/readers/sqldir"
|
"git.warky.dev/wdevs/relspecgo/pkg/readers/sqldir"
|
||||||
|
"git.warky.dev/wdevs/relspecgo/pkg/transform"
|
||||||
"git.warky.dev/wdevs/relspecgo/pkg/writers"
|
"git.warky.dev/wdevs/relspecgo/pkg/writers"
|
||||||
wpgsql "git.warky.dev/wdevs/relspecgo/pkg/writers/pgsql"
|
wpgsql "git.warky.dev/wdevs/relspecgo/pkg/writers/pgsql"
|
||||||
"git.warky.dev/wdevs/relspecgo/pkg/writers/sqlexec"
|
"git.warky.dev/wdevs/relspecgo/pkg/writers/sqlexec"
|
||||||
@@ -522,6 +523,12 @@ func runTemplJob(rj *resolvedJob, lg *jobLogger) error {
|
|||||||
return fmt.Errorf("schema not found: %s", schema)
|
return fmt.Errorf("schema not found: %s", schema)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
if lowercaseIdentifiers || rj.job.Options.LowercaseIdentifiers {
|
||||||
|
db, err = transform.LowercaseIdentifiers(db)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
mode := rj.job.Mode
|
mode := rj.job.Mode
|
||||||
if mode == "" {
|
if mode == "" {
|
||||||
mode = "database"
|
mode = "database"
|
||||||
@@ -777,6 +784,8 @@ func runScriptsExecJob(rj *resolvedJob, lg *jobLogger) error {
|
|||||||
lg.logf("executing %d script(s) against database env:%s", len(combined.Scripts), rj.outputConnEnv)
|
lg.logf("executing %d script(s) against database env:%s", len(combined.Scripts), rj.outputConnEnv)
|
||||||
|
|
||||||
writer := sqlexec.NewWriter(&writers.WriterOptions{
|
writer := sqlexec.NewWriter(&writers.WriterOptions{
|
||||||
|
ContinueOnError: rj.job.Options.ContinueOnError,
|
||||||
|
AtomicExecution: rj.job.Options.AtomicExecution,
|
||||||
Metadata: map[string]any{
|
Metadata: map[string]any{
|
||||||
"connection_string": rj.outputConn,
|
"connection_string": rj.outputConn,
|
||||||
"ignore_errors": rj.job.Options.ContinueOnError,
|
"ignore_errors": rj.job.Options.ContinueOnError,
|
||||||
@@ -791,7 +800,7 @@ func runScriptsExecJob(rj *resolvedJob, lg *jobLogger) error {
|
|||||||
success, _ := opts.Metadata["execution_success"].(int)
|
success, _ := opts.Metadata["execution_success"].(int)
|
||||||
failed, _ := opts.Metadata["execution_failed"].(int)
|
failed, _ := opts.Metadata["execution_failed"].(int)
|
||||||
lg.logf("executed %d script(s): %d succeeded, %d failed", total, success, failed)
|
lg.logf("executed %d script(s): %d succeeded, %d failed", total, success, failed)
|
||||||
if failed > 0 && !rj.job.Options.ContinueOnError {
|
if failed > 0 {
|
||||||
return fmt.Errorf("%d script(s) failed", failed)
|
return fmt.Errorf("%d script(s) failed", failed)
|
||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
@@ -835,7 +844,17 @@ func inputLabel(ri resolvedInput) string {
|
|||||||
|
|
||||||
// writeJobOutput writes db to the job's output target (file or database).
|
// writeJobOutput writes db to the job's output target (file or database).
|
||||||
func writeJobOutput(rj *resolvedJob, db *models.Database, lg *jobLogger) error {
|
func writeJobOutput(rj *resolvedJob, db *models.Database, lg *jobLogger) error {
|
||||||
|
if lowercaseIdentifiers || rj.job.Options.LowercaseIdentifiers {
|
||||||
|
var err error
|
||||||
|
db, err = transform.LowercaseIdentifiers(db)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
o := rj.job.Options
|
o := rj.job.Options
|
||||||
|
if lowercaseIdentifiers || o.LowercaseIdentifiers {
|
||||||
|
o.Schema = strings.ToLower(o.Schema)
|
||||||
|
}
|
||||||
format := strings.ToLower(rj.job.Output.Format)
|
format := strings.ToLower(rj.job.Output.Format)
|
||||||
|
|
||||||
if rj.outputConn != "" {
|
if rj.outputConn != "" {
|
||||||
@@ -844,6 +863,7 @@ func writeJobOutput(rj *resolvedJob, db *models.Database, lg *jobLogger) error {
|
|||||||
}
|
}
|
||||||
lg.logf("writing output to database env:%s", rj.outputConnEnv)
|
lg.logf("writing output to database env:%s", rj.outputConnEnv)
|
||||||
writerOpts := newWriterOptions("", o.Package, o.FlattenSchema, o.Types, o.ArrayNullable, o.ContinueOnError)
|
writerOpts := newWriterOptions("", o.Package, o.FlattenSchema, o.Types, o.ArrayNullable, o.ContinueOnError)
|
||||||
|
writerOpts.AtomicExecution = o.AtomicExecution
|
||||||
writerOpts.Metadata = map[string]interface{}{"connection_string": rj.outputConn, "full_ddl": o.FullDDL}
|
writerOpts.Metadata = map[string]interface{}{"connection_string": rj.outputConn, "full_ddl": o.FullDDL}
|
||||||
return wpgsql.NewWriter(writerOpts).WriteDatabase(db)
|
return wpgsql.NewWriter(writerOpts).WriteDatabase(db)
|
||||||
}
|
}
|
||||||
@@ -854,7 +874,7 @@ func writeJobOutput(rj *resolvedJob, db *models.Database, lg *jobLogger) error {
|
|||||||
lg.logf("writing output: %s (%s)", rj.outputPath, format)
|
lg.logf("writing output: %s (%s)", rj.outputPath, format)
|
||||||
|
|
||||||
write := func(target string) error {
|
write := func(target string) error {
|
||||||
return writeDatabase(db, format, target, o.Package, o.Schema, o.FlattenSchema, o.Types, o.ArrayNullable, o.ContinueOnError, "")
|
return writeDatabase(db, format, target, o.Package, o.Schema, o.FlattenSchema, o.Types, o.ArrayNullable, o.ContinueOnError, "", o.StrictConversion)
|
||||||
}
|
}
|
||||||
// Single-file formats are written to a temp file and renamed into place so
|
// Single-file formats are written to a temp file and renamed into place so
|
||||||
// a failure never leaves a partial or truncated output. Directory-emitting
|
// a failure never leaves a partial or truncated output. Directory-emitting
|
||||||
|
|||||||
@@ -2,11 +2,15 @@ package main
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"bytes"
|
"bytes"
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/jackc/pgx/v5"
|
||||||
"github.com/spf13/cobra"
|
"github.com/spf13/cobra"
|
||||||
|
|
||||||
"git.warky.dev/wdevs/relspecgo/pkg/jobs"
|
"git.warky.dev/wdevs/relspecgo/pkg/jobs"
|
||||||
@@ -741,3 +745,104 @@ jobs:
|
|||||||
t.Fatalf("expected templ format rejection, got %v", err)
|
t.Fatalf("expected templ format rejection, got %v", err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestLive_JobPostgresFailureStopsDependents(t *testing.T) {
|
||||||
|
connString := os.Getenv("RELSPEC_TEST_PG_CONN")
|
||||||
|
if connString == "" {
|
||||||
|
t.Skip("RELSPEC_TEST_PG_CONN not set")
|
||||||
|
}
|
||||||
|
t.Setenv("RELSPEC_TEST_JOB_OUTPUT_CONN", connString)
|
||||||
|
for _, continueOnError := range []string{"false", "true"} {
|
||||||
|
t.Run("continue="+continueOnError, func(t *testing.T) {
|
||||||
|
manifest := `version: 1
|
||||||
|
jobs:
|
||||||
|
apply:
|
||||||
|
command: convert
|
||||||
|
inputs:
|
||||||
|
- path: schema/core.dbml
|
||||||
|
format: dbml
|
||||||
|
output:
|
||||||
|
format: pgsql
|
||||||
|
conn_env: RELSPEC_TEST_JOB_OUTPUT_CONN
|
||||||
|
options:
|
||||||
|
full_ddl: true
|
||||||
|
continue_on_error: ` + continueOnError + `
|
||||||
|
logfile: .relspec/apply.log
|
||||||
|
dependent:
|
||||||
|
command: convert
|
||||||
|
depends_on: [apply]
|
||||||
|
inputs:
|
||||||
|
- path: schema/core.dbml
|
||||||
|
format: dbml
|
||||||
|
output:
|
||||||
|
format: json
|
||||||
|
path: build/dependent.json
|
||||||
|
logfile: .relspec/dependent.log
|
||||||
|
`
|
||||||
|
dir := jobFixture(t, manifest)
|
||||||
|
schema := fmt.Sprintf("job_audit_%d", time.Now().UnixNano())
|
||||||
|
conn, err := pgx.Connect(context.Background(), connString)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
t.Cleanup(func() {
|
||||||
|
_, _ = conn.Exec(context.Background(), "DROP SCHEMA IF EXISTS "+schema+" CASCADE")
|
||||||
|
_ = conn.Close(context.Background())
|
||||||
|
})
|
||||||
|
// An invalid column type prevents table creation in this isolated schema.
|
||||||
|
writeFile(t, filepath.Join(dir, "schema", "core.dbml"), "Table "+schema+".users {\n id no_such_type_audit_xyz\n}\n")
|
||||||
|
set := mustLoadSet(t, filepath.Join(dir, "relspec.yml"))
|
||||||
|
err = executeJobPlan(set, "dependent", false, false, &bytes.Buffer{})
|
||||||
|
if err == nil || !strings.Contains(err.Error(), `job "apply" failed`) {
|
||||||
|
t.Fatalf("expected database failure propagation, got %v", err)
|
||||||
|
}
|
||||||
|
for _, path := range []string{"build/dependent.json", ".relspec/dependent.log"} {
|
||||||
|
if _, err := os.Stat(filepath.Join(dir, path)); !os.IsNotExist(err) {
|
||||||
|
t.Fatalf("dependent must not start: path=%s err=%v", path, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
logData, err := os.ReadFile(filepath.Join(dir, ".relspec/apply.log"))
|
||||||
|
if err != nil || !strings.Contains(string(logData), "FAILED") {
|
||||||
|
t.Fatalf("failed application must be logged: %s err=%v", logData, err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestLive_JobScriptContinuationStillStopsDependents(t *testing.T) {
|
||||||
|
connection := os.Getenv("RELSPEC_TEST_PG_CONN")
|
||||||
|
if connection == "" {
|
||||||
|
t.Skip("RELSPEC_TEST_PG_CONN not set")
|
||||||
|
}
|
||||||
|
t.Setenv("RELSPEC_TEST_EXEC_CONN", connection)
|
||||||
|
dir := jobFixture(t, `version: 1
|
||||||
|
jobs:
|
||||||
|
migrate:
|
||||||
|
command: scripts-exec
|
||||||
|
script_dirs: [migrations]
|
||||||
|
output:
|
||||||
|
conn_env: RELSPEC_TEST_EXEC_CONN
|
||||||
|
options:
|
||||||
|
continue_on_error: true
|
||||||
|
logfile: .relspec/migrate.log
|
||||||
|
dependent:
|
||||||
|
command: convert
|
||||||
|
depends_on: [migrate]
|
||||||
|
inputs:
|
||||||
|
- path: schema/core.dbml
|
||||||
|
format: dbml
|
||||||
|
output:
|
||||||
|
format: json
|
||||||
|
path: build/dependent.json
|
||||||
|
`)
|
||||||
|
writeFile(t, filepath.Join(dir, "migrations", "1_001_bad.sql"), "SELECT 1 / 0;\n")
|
||||||
|
writeFile(t, filepath.Join(dir, "migrations", "1_002_good.sql"), "SELECT 1;\n")
|
||||||
|
set := mustLoadSet(t, filepath.Join(dir, "relspec.yml"))
|
||||||
|
err := executeJobPlan(set, "dependent", false, false, &bytes.Buffer{})
|
||||||
|
if err == nil || !strings.Contains(err.Error(), `job "migrate" failed`) {
|
||||||
|
t.Fatalf("failed scripts must fail job: %v", err)
|
||||||
|
}
|
||||||
|
if _, err := os.Stat(filepath.Join(dir, "build", "dependent.json")); !os.IsNotExist(err) {
|
||||||
|
t.Fatalf("dependent ran: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -0,0 +1,82 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"git.warky.dev/wdevs/relspecgo/pkg/models"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestLowercaseFlagAndFileOutput(t *testing.T) {
|
||||||
|
prior := lowercaseIdentifiers
|
||||||
|
t.Cleanup(func() { lowercaseIdentifiers = prior })
|
||||||
|
for _, cmd := range []struct {
|
||||||
|
name string
|
||||||
|
flag bool
|
||||||
|
}{{"convert", convertCmd.Flags().Lookup("lowercase") != nil}, {"batch", batchCmd.Flags().Lookup("lowercase") != nil}, {"merge", mergeCmd.Flags().Lookup("lowercase") != nil}, {"templ", templCmd.Flags().Lookup("lowercase") != nil}, {"job", jobCmd.PersistentFlags().Lookup("lowercase") != nil}} {
|
||||||
|
if !cmd.flag {
|
||||||
|
t.Fatalf("missing %s --lowercase", cmd.name)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if err := convertCmd.Flags().Set("lowercase", "true"); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
path := filepath.Join(t.TempDir(), "schema.sql")
|
||||||
|
db := models.InitDatabase("DB")
|
||||||
|
s := models.InitSchema("App")
|
||||||
|
table := models.InitTable("Users", "App")
|
||||||
|
col := models.InitColumn("Name", "Users", "App")
|
||||||
|
col.Type = "text"
|
||||||
|
col.Default = "Keep CASE"
|
||||||
|
table.Columns[col.Name] = col
|
||||||
|
s.Tables = append(s.Tables, table)
|
||||||
|
db.Schemas = append(db.Schemas, s)
|
||||||
|
if err := writeDatabase(db, "pgsql", path, "", "", false, "", "", false, ""); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
data, err := os.ReadFile(path)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if !strings.Contains(string(data), `"app"."users"`) || !strings.Contains(string(data), `"name" text DEFAULT 'Keep CASE'`) {
|
||||||
|
t.Fatalf("%s", data)
|
||||||
|
}
|
||||||
|
if table.Name != "Users" {
|
||||||
|
t.Fatal("input mutated")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestJobLowercaseOption(t *testing.T) {
|
||||||
|
prior := lowercaseIdentifiers
|
||||||
|
lowercaseIdentifiers = false
|
||||||
|
t.Cleanup(func() { lowercaseIdentifiers = prior })
|
||||||
|
dir := t.TempDir()
|
||||||
|
writeTestJSON(t, filepath.Join(dir, "input.json"), []string{"MixedCase"})
|
||||||
|
writeFile(t, filepath.Join(dir, "relspec.yml"), `version: 1
|
||||||
|
jobs:
|
||||||
|
lower:
|
||||||
|
command: convert
|
||||||
|
inputs:
|
||||||
|
- format: json
|
||||||
|
path: input.json
|
||||||
|
output:
|
||||||
|
format: json
|
||||||
|
path: output.json
|
||||||
|
options:
|
||||||
|
lowercase_identifiers: true
|
||||||
|
`)
|
||||||
|
set := mustLoadSet(t, filepath.Join(dir, "relspec.yml"))
|
||||||
|
if err := executeJobPlan(set, "lower", false, false, &bytes.Buffer{}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
data, err := os.ReadFile(filepath.Join(dir, "output.json"))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if !strings.Contains(string(data), "mixedcase") || strings.Contains(string(data), "MixedCase") {
|
||||||
|
t.Fatalf("%s", data)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -24,6 +24,7 @@ import (
|
|||||||
"git.warky.dev/wdevs/relspecgo/pkg/readers/sqlite"
|
"git.warky.dev/wdevs/relspecgo/pkg/readers/sqlite"
|
||||||
"git.warky.dev/wdevs/relspecgo/pkg/readers/typeorm"
|
"git.warky.dev/wdevs/relspecgo/pkg/readers/typeorm"
|
||||||
"git.warky.dev/wdevs/relspecgo/pkg/readers/yaml"
|
"git.warky.dev/wdevs/relspecgo/pkg/readers/yaml"
|
||||||
|
"git.warky.dev/wdevs/relspecgo/pkg/transform"
|
||||||
"git.warky.dev/wdevs/relspecgo/pkg/writers"
|
"git.warky.dev/wdevs/relspecgo/pkg/writers"
|
||||||
wbun "git.warky.dev/wdevs/relspecgo/pkg/writers/bun"
|
wbun "git.warky.dev/wdevs/relspecgo/pkg/writers/bun"
|
||||||
wdbml "git.warky.dev/wdevs/relspecgo/pkg/writers/dbml"
|
wdbml "git.warky.dev/wdevs/relspecgo/pkg/writers/dbml"
|
||||||
@@ -407,6 +408,14 @@ func readDatabaseForMerge(dbType, filePath, connString, label string) (*models.D
|
|||||||
}
|
}
|
||||||
|
|
||||||
func writeDatabaseForMerge(dbType, filePath, connString string, db *models.Database, label string, flattenSchema bool) error {
|
func writeDatabaseForMerge(dbType, filePath, connString string, db *models.Database, label string, flattenSchema bool) error {
|
||||||
|
if lowercaseIdentifiers {
|
||||||
|
var err error
|
||||||
|
db, err = transform.LowercaseIdentifiers(db)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
var writer writers.Writer
|
var writer writers.Writer
|
||||||
|
|
||||||
switch strings.ToLower(dbType) {
|
switch strings.ToLower(dbType) {
|
||||||
|
|||||||
@@ -25,11 +25,13 @@ func newWriterOptions(outputPath, packageName string, flattenSchema bool, nullab
|
|||||||
OutputPath: outputPath,
|
OutputPath: outputPath,
|
||||||
PackageName: packageName,
|
PackageName: packageName,
|
||||||
FlattenSchema: flattenSchema,
|
FlattenSchema: flattenSchema,
|
||||||
|
LowercaseIdentifiers: lowercaseIdentifiers,
|
||||||
NullableTypes: nullableTypes,
|
NullableTypes: nullableTypes,
|
||||||
NullableArrays: nullableArrays,
|
NullableArrays: nullableArrays,
|
||||||
TypeMappings: typeMappings,
|
TypeMappings: typeMappings,
|
||||||
Prisma7: prisma7,
|
Prisma7: prisma7,
|
||||||
ContinueOnError: continueOnError,
|
ContinueOnError: continueOnError,
|
||||||
StrictDirectives: strictDirectives,
|
StrictDirectives: strictDirectives,
|
||||||
|
StrictConversion: strictConversion,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -18,6 +18,8 @@ var (
|
|||||||
noVersion bool
|
noVersion bool
|
||||||
silent bool
|
silent bool
|
||||||
strictDirectives bool
|
strictDirectives bool
|
||||||
|
strictConversion bool
|
||||||
|
lowercaseIdentifiers bool
|
||||||
typeMapFlags []string
|
typeMapFlags []string
|
||||||
typeMappings map[string]string
|
typeMappings map[string]string
|
||||||
)
|
)
|
||||||
@@ -39,12 +41,16 @@ JSON, YAML, SQL, etc.).`,
|
|||||||
}
|
}
|
||||||
|
|
||||||
func init() {
|
func init() {
|
||||||
|
for _, cmd := range []*cobra.Command{convertCmd, batchCmd, mergeCmd, splitCmd, templCmd} {
|
||||||
|
cmd.Flags().BoolVar(&lowercaseIdentifiers, "lowercase", false, "Lowercase all model identifiers and structured references before output (SQL expressions and data keep their case)")
|
||||||
|
}
|
||||||
rootCmd.AddCommand(convertCmd)
|
rootCmd.AddCommand(convertCmd)
|
||||||
rootCmd.AddCommand(batchCmd)
|
rootCmd.AddCommand(batchCmd)
|
||||||
rootCmd.AddCommand(diffCmd)
|
rootCmd.AddCommand(diffCmd)
|
||||||
rootCmd.AddCommand(inspectCmd)
|
rootCmd.AddCommand(inspectCmd)
|
||||||
rootCmd.AddCommand(scriptsCmd)
|
rootCmd.AddCommand(scriptsCmd)
|
||||||
rootCmd.AddCommand(jobCmd)
|
rootCmd.AddCommand(jobCmd)
|
||||||
|
jobCmd.PersistentFlags().BoolVar(&lowercaseIdentifiers, "lowercase", false, "Lowercase all model identifiers and structured references before schema output")
|
||||||
rootCmd.AddCommand(assetsCmd)
|
rootCmd.AddCommand(assetsCmd)
|
||||||
rootCmd.AddCommand(templCmd)
|
rootCmd.AddCommand(templCmd)
|
||||||
rootCmd.AddCommand(editCmd)
|
rootCmd.AddCommand(editCmd)
|
||||||
@@ -57,6 +63,7 @@ func init() {
|
|||||||
rootCmd.PersistentFlags().StringArrayVar(&typeMapFlags, "type-map", nil, "Override a SQL-to-Go type mapping for bun/gorm output as sqltype=gotype (repeatable), e.g. --type-map uuid=uuid.UUID --type-map numeric=decimal.Decimal")
|
rootCmd.PersistentFlags().StringArrayVar(&typeMapFlags, "type-map", nil, "Override a SQL-to-Go type mapping for bun/gorm output as sqltype=gotype (repeatable), e.g. --type-map uuid=uuid.UUID --type-map numeric=decimal.Decimal")
|
||||||
rootCmd.PersistentFlags().BoolVar(&noVersion, "no-version", false, "Suppress the RelSpec version header")
|
rootCmd.PersistentFlags().BoolVar(&noVersion, "no-version", false, "Suppress the RelSpec version header")
|
||||||
rootCmd.PersistentFlags().BoolVar(&silent, "silent", false, "Suppress progress and status messages (errors are still shown)")
|
rootCmd.PersistentFlags().BoolVar(&silent, "silent", false, "Suppress progress and status messages (errors are still shown)")
|
||||||
|
rootCmd.PersistentFlags().BoolVar(&strictConversion, "strict-conversion", false, "Reject known lossy MySQL schema conversions before writing or executing output")
|
||||||
rootCmd.PersistentFlags().BoolVar(&strictDirectives, "strict-directives", false, "Fail on unknown or untranslatable DBML dialect directives (@postgres:, @sqlite:, …)")
|
rootCmd.PersistentFlags().BoolVar(&strictDirectives, "strict-directives", false, "Fail on unknown or untranslatable DBML dialect directives (@postgres:, @sqlite:, …)")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -63,7 +63,7 @@ var scriptsExecuteCmd = &cobra.Command{
|
|||||||
Long: `Execute SQL scripts from a directory against a PostgreSQL database.
|
Long: `Execute SQL scripts from a directory against a PostgreSQL database.
|
||||||
|
|
||||||
Scripts are executed in order: Priority (ascending), Sequence (ascending), Name (alphabetical).
|
Scripts are executed in order: Priority (ascending), Sequence (ascending), Name (alphabetical).
|
||||||
By default, execution stops immediately on the first error. Use --ignore-errors to continue execution.
|
By default, execution stops immediately on the first error. Use --ignore-errors (or --continue-on-error / --coe) to attempt later scripts; any failure still exits nonzero.
|
||||||
|
|
||||||
The directory is scanned recursively for all subdirectories and files matching the patterns:
|
The directory is scanned recursively for all subdirectories and files matching the patterns:
|
||||||
{priority}_{sequence}_{name}.sql or .pgsql (underscore format)
|
{priority}_{sequence}_{name}.sql or .pgsql (underscore format)
|
||||||
@@ -112,6 +112,8 @@ func init() {
|
|||||||
scriptsExecuteCmd.Flags().StringVar(&scriptsSchemaName, "schema", "public", "Schema name (optional, default: public)")
|
scriptsExecuteCmd.Flags().StringVar(&scriptsSchemaName, "schema", "public", "Schema name (optional, default: public)")
|
||||||
scriptsExecuteCmd.Flags().StringVar(&scriptsDBName, "database", "database", "Database name (optional, default: database)")
|
scriptsExecuteCmd.Flags().StringVar(&scriptsDBName, "database", "database", "Database name (optional, default: database)")
|
||||||
scriptsExecuteCmd.Flags().BoolVar(&scriptsIgnoreErrors, "ignore-errors", false, "Continue executing scripts even if errors occur")
|
scriptsExecuteCmd.Flags().BoolVar(&scriptsIgnoreErrors, "ignore-errors", false, "Continue executing scripts even if errors occur")
|
||||||
|
scriptsExecuteCmd.Flags().BoolVar(&scriptsIgnoreErrors, "continue-on-error", false, "Attempt later scripts after failures; still exit nonzero")
|
||||||
|
scriptsExecuteCmd.Flags().BoolVar(&scriptsIgnoreErrors, "coe", false, "Alias for --continue-on-error")
|
||||||
|
|
||||||
err = scriptsExecuteCmd.MarkFlagRequired("dir")
|
err = scriptsExecuteCmd.MarkFlagRequired("dir")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|||||||
@@ -0,0 +1,41 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"io"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"git.warky.dev/wdevs/relspecgo/pkg/jobs"
|
||||||
|
"git.warky.dev/wdevs/relspecgo/pkg/models"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestStrictConversionOptionAndDryRun(t *testing.T) {
|
||||||
|
prior := strictConversion
|
||||||
|
strictConversion = true
|
||||||
|
t.Cleanup(func() { strictConversion = prior })
|
||||||
|
if flag := rootCmd.PersistentFlags().Lookup("strict-conversion"); flag == nil {
|
||||||
|
t.Fatal("missing --strict-conversion")
|
||||||
|
}
|
||||||
|
db := models.InitDatabase("shop")
|
||||||
|
s := models.InitSchema("shop")
|
||||||
|
table := models.InitTable("users", "shop")
|
||||||
|
col := models.InitColumn("value", "users", "shop")
|
||||||
|
col.Type = "unknown_type"
|
||||||
|
table.Columns[col.Name] = col
|
||||||
|
s.Tables = append(s.Tables, table)
|
||||||
|
db.Schemas = append(db.Schemas, s)
|
||||||
|
if err := validateWriteTarget(db, "mysql", "", "", ""); err == nil || !strings.Contains(err.Error(), "lossy MySQL") {
|
||||||
|
t.Fatalf("dry run bypassed strict validation: %v", err)
|
||||||
|
}
|
||||||
|
strictConversion = false
|
||||||
|
out := filepath.Join(t.TempDir(), "out.sql")
|
||||||
|
rj := &resolvedJob{job: &jobs.Job{Output: &jobs.Output{Format: "mysql"}, Options: jobs.Options{StrictConversion: true}}, outputPath: out}
|
||||||
|
if err := writeJobOutput(rj, db, &jobLogger{file: io.Discard}); err == nil || !strings.Contains(err.Error(), "lossy MySQL") {
|
||||||
|
t.Fatalf("job option bypassed strict validation: %v", err)
|
||||||
|
}
|
||||||
|
if _, err := os.Stat(out); !os.IsNotExist(err) {
|
||||||
|
t.Fatalf("strict job created output: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -7,6 +7,7 @@ import (
|
|||||||
"github.com/spf13/cobra"
|
"github.com/spf13/cobra"
|
||||||
|
|
||||||
"git.warky.dev/wdevs/relspecgo/pkg/models"
|
"git.warky.dev/wdevs/relspecgo/pkg/models"
|
||||||
|
"git.warky.dev/wdevs/relspecgo/pkg/transform"
|
||||||
"git.warky.dev/wdevs/relspecgo/pkg/writers"
|
"git.warky.dev/wdevs/relspecgo/pkg/writers"
|
||||||
wtemplate "git.warky.dev/wdevs/relspecgo/pkg/writers/template"
|
wtemplate "git.warky.dev/wdevs/relspecgo/pkg/writers/template"
|
||||||
)
|
)
|
||||||
@@ -142,6 +143,13 @@ func runTempl(cmd *cobra.Command, args []string) error {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if lowercaseIdentifiers {
|
||||||
|
db, err = transform.LowercaseIdentifiers(db)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// Create template writer
|
// Create template writer
|
||||||
fmt.Fprintf(os.Stderr, "Loading template: %s\n", templTemplatePath)
|
fmt.Fprintf(os.Stderr, "Loading template: %s\n", templTemplatePath)
|
||||||
fmt.Fprintf(os.Stderr, "Execution mode: %s\n", templMode)
|
fmt.Fprintf(os.Stderr, "Execution mode: %s\n", templMode)
|
||||||
|
|||||||
@@ -191,6 +191,8 @@ jobs:
|
|||||||
# (file output = full DDL); set options.full_ddl to force full DDL
|
# (file output = full DDL); set options.full_ddl to force full DDL
|
||||||
overwrite: false # default false
|
overwrite: false # default false
|
||||||
options:
|
options:
|
||||||
|
strict_conversion: false # reject known lossy MySQL output before writing
|
||||||
|
lowercase_identifiers: false # lowercase names/references; SQL expressions and data keep their case
|
||||||
flatten_schema: false
|
flatten_schema: false
|
||||||
schema: public
|
schema: public
|
||||||
package: models # for gorm/bun output
|
package: models # for gorm/bun output
|
||||||
|
|||||||
@@ -0,0 +1,131 @@
|
|||||||
|
# MySQL schema fidelity
|
||||||
|
|
||||||
|
The MySQL reader/writer preserves the table-schema features below. It generates
|
||||||
|
DDL for **new tables in existing databases**, not migrations or data copies.
|
||||||
|
MySQL 8.4 / InnoDB is exercised by the live suite. The reader uses MySQL 8.0.23+
|
||||||
|
metadata (including functional indexes, checks, spatial metadata, and invisible
|
||||||
|
columns); older MySQL versions and the `mariadb` alias are not certified by this
|
||||||
|
suite and may return explicit metadata-query errors.
|
||||||
|
|
||||||
|
## Conversion diagnostics
|
||||||
|
|
||||||
|
MySQL output warns on stderr when it encounters known unsupported features.
|
||||||
|
Add `--strict-conversion` to reject these conversions before opening an output
|
||||||
|
file or connecting to an output database:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
relspec convert --from mysql --from-conn "$MYSQL_DSN" \
|
||||||
|
--to mysql --to-path schema.sql --strict-conversion
|
||||||
|
|
||||||
|
relspec convert --from json --from-path schema.json \
|
||||||
|
--to mysql --to-path schema.sql --strict-conversion --dry-run
|
||||||
|
```
|
||||||
|
|
||||||
|
The flag also applies to MySQL output from `batch`, `split`, and schema jobs.
|
||||||
|
Jobs can set `options.strict_conversion: true`. Library callers use
|
||||||
|
`writers.WriterOptions.StrictConversion`; `Warning func(string)` collects
|
||||||
|
non-strict diagnostics instead of printing them. `mysql.Writer.Statements`
|
||||||
|
performs the same preflight without opening a destination and returns structured
|
||||||
|
SQL statements suitable for execution without splitting on semicolons.
|
||||||
|
|
||||||
|
Strict conversion currently covers **MySQL output**. It is not a general
|
||||||
|
losslessness check for other writers or a validator of arbitrary SQL fragments.
|
||||||
|
Warnings identify the object and omitted or changed feature. Inspect them before
|
||||||
|
using non-strict output; unsupported index types are omitted, while a partial
|
||||||
|
index's predicate and INCLUDE columns cannot be represented in MySQL.
|
||||||
|
|
||||||
|
## MySQL capability matrix
|
||||||
|
|
||||||
|
| Feature | Reader → model | Model → MySQL DDL |
|
||||||
|
| --- | --- | --- |
|
||||||
|
| Ordered columns, nullability, primary/composite keys | Preserved | Preserved |
|
||||||
|
| Unique/composite keys | Preserved, including backing indexes | Preserved once; no duplicate unique index |
|
||||||
|
| Foreign/composite keys and DELETE/UPDATE actions | Preserved | Added after all tables, allowing forward references/cycles |
|
||||||
|
| CHECK and NOT ENFORCED CHECK | Preserved | Preserved |
|
||||||
|
| BTREE, FULLTEXT, SPATIAL indexes | Preserved | Emitted; engine restrictions still apply |
|
||||||
|
| Expression/prefix/descending/invisible index parts | Preserved in `Index.KeyParts` and metadata | Preserved |
|
||||||
|
| HASH indexes | Preserved | Engine-dependent; incompatible engine produces a diagnostic |
|
||||||
|
| Integer unsignedness; mediumint; enum/set; binary/blob/text variants | Canonical type plus native declaration | Native declaration preserved |
|
||||||
|
| Length, numeric precision/scale, fractional seconds | Typed dimensions plus native declaration | Preserved; canonical dimensions also supported |
|
||||||
|
| Defaults, including SQL expressions, bit/binary literals and multiline text | Values plus native SQL spelling from SHOW CREATE | Preserved |
|
||||||
|
| Auto-increment and current next value | Preserved | Preserved |
|
||||||
|
| Stored/virtual generated columns, ON UPDATE, invisible columns | Preserved | Preserved |
|
||||||
|
| Engine, table/column collation, charset and comments | Preserved | Preserved |
|
||||||
|
| Views, triggers, routines, events, partitioned tables | Names recorded in `mysql_unsupported` | Diagnostic; bodies/partition definitions are not emitted |
|
||||||
|
| Nonempty table CREATE_OPTIONS; native parser/storage/engine attributes; spatial SRID | Recorded as unsupported attributes | Diagnostic; not emitted |
|
||||||
|
| Partial/INCLUDE indexes, unsupported index methods | Retained when supplied in the model | Diagnostic |
|
||||||
|
| Deferrable constraints, identity semantics, timestamptz, jsonb storage | Retained when supplied in the model | Diagnostic for unsupported/different semantics |
|
||||||
|
| Standalone sequences/enums/domains, ownership/permissions, scripts and dialect directives | Retained when supplied in the model | Diagnostic; not emitted |
|
||||||
|
| Data, grants, users, server/plugin configuration | Outside the reader's schema contract | Outside the writer's contract |
|
||||||
|
|
||||||
|
Diagnostics depend on the objects visible to the connection account. They cover
|
||||||
|
known modeled/discovered features, not every engine/plugin extension. The server
|
||||||
|
still validates SQL expressions, engine rules, and references at execution time.
|
||||||
|
For example, source and target engines must support the constraints requested.
|
||||||
|
|
||||||
|
## Preservation across intermediate formats
|
||||||
|
|
||||||
|
| Path | Scope |
|
||||||
|
| --- | --- |
|
||||||
|
| MySQL → model → MySQL | Live-tested for the features above on MySQL 8.4 / InnoDB |
|
||||||
|
| MySQL → JSON → model → MySQL | Live-tested; preserves typed index parts and MySQL metadata |
|
||||||
|
| MySQL → YAML → model → MySQL | YAML serializes the same fields/metadata; covered by the live round-trip test |
|
||||||
|
| Other source/target formats | Their own mappings apply; MySQL-specific metadata and semantics are not certified to survive |
|
||||||
|
|
||||||
|
Native declarations and defaults are stored in column metadata as `mysql_type`
|
||||||
|
and `mysql_default_sql`. Reader-generated signatures (`mysql_type_signature`,
|
||||||
|
`mysql_default_value`) prevent these spellings from silently overriding edited
|
||||||
|
types/dimensions/defaults. Editing canonical dimensions invalidates the native
|
||||||
|
type signature, produces a diagnostic, and uses the canonical mapping instead.
|
||||||
|
For a binary default, the logical default is its SQL hex/literal spelling so it
|
||||||
|
survives serialization without invalid UTF-8.
|
||||||
|
|
||||||
|
Other metadata includes `mysql_charset`, `mysql_generation_storage`,
|
||||||
|
`mysql_default_expression`, `mysql_on_update`, `mysql_invisible`, table
|
||||||
|
`mysql_engine` / `mysql_collation` / `mysql_auto_increment`, index
|
||||||
|
`mysql_visible`, and check `mysql_enforced`. Unknown `mysql_*` metadata produces
|
||||||
|
a diagnostic. For a manually authored expression default, set
|
||||||
|
`mysql_default_expression: true`; otherwise strings are literal values.
|
||||||
|
`Index.KeyParts` is authoritative when present; `Index.Columns` remains the
|
||||||
|
compatibility representation for simple indexes.
|
||||||
|
|
||||||
|
Raw native type declarations, default SQL, generation/check/index expressions,
|
||||||
|
and ON UPDATE expressions are executable SQL input. Lowercasing changes names
|
||||||
|
and structured index parts but leaves expressions untouched; update references
|
||||||
|
inside raw SQL yourself when using `--lowercase`.
|
||||||
|
|
||||||
|
## Applying the output
|
||||||
|
|
||||||
|
Create the destination database/schema first. Qualified names are preserved,
|
||||||
|
including foreign-key target schemas. Adjust them explicitly when restoring to
|
||||||
|
a different database. Output assumes standard MySQL backslash escaping;
|
||||||
|
`NO_BACKSLASH_ESCAPES` or `ANSI_QUOTES` on the source produces a conversion
|
||||||
|
diagnostic, and live execution refuses these target modes before DDL. File consumers must use the same
|
||||||
|
supported mode. Other SQL-mode incompatibilities surface as server errors.
|
||||||
|
|
||||||
|
`CREATE TABLE IF NOT EXISTS` does not reconcile existing definitions. Foreign
|
||||||
|
keys use `ALTER TABLE ADD CONSTRAINT` and therefore repeated application can
|
||||||
|
fail on existing constraints. This is creation DDL, not a rerunnable migration.
|
||||||
|
Live execution stops on the first SQL error by default. Explicit
|
||||||
|
`ContinueOnError` attempts later statements and still returns an error if any
|
||||||
|
fail. MySQL DDL implicitly commits; `AtomicExecution` is rejected. PostgreSQL's
|
||||||
|
shared execution-report support is not yet implemented for this writer.
|
||||||
|
|
||||||
|
## Verification
|
||||||
|
|
||||||
|
```bash
|
||||||
|
./tests/dbtest/dbtest.sh test mysql
|
||||||
|
# Or use your own disposable database:
|
||||||
|
RELSPEC_TEST_MYSQL_CONN="$MYSQL_DSN" go test -mod=vendor ./pkg/readers/mysql ./pkg/writers/mysql
|
||||||
|
```
|
||||||
|
|
||||||
|
The live tests compare MySQL's `SHOW CREATE TABLE` definitions, normalizing only
|
||||||
|
temporary target-name suffixes and nonsemantic ordinary-index display order.
|
||||||
|
They re-read the target, exercise constraint enforcement/default values, and
|
||||||
|
verify unsupported objects produce warnings or strict failures without touching
|
||||||
|
an existing output file. Test fixtures are removed after each run.
|
||||||
|
|
||||||
|
Metadata references: [COLUMNS](https://dev.mysql.com/doc/refman/8.4/en/information-schema-columns-table.html),
|
||||||
|
[STATISTICS](https://dev.mysql.com/doc/refman/8.4/en/information-schema-statistics-table.html),
|
||||||
|
[TABLE_CONSTRAINTS](https://dev.mysql.com/doc/refman/8.4/en/information-schema-table-constraints-table.html),
|
||||||
|
and [REFERENTIAL_CONSTRAINTS](https://dev.mysql.com/doc/refman/8.4/en/information-schema-referential-constraints-table.html).
|
||||||
+30
-2
@@ -96,11 +96,39 @@ INSERT INTO assets (message, payload)
|
|||||||
VALUES (:message, decode(:payload, 'base64')::bytea);
|
VALUES (:message, decode(:payload, 'base64')::bytea);
|
||||||
```
|
```
|
||||||
|
|
||||||
- `path`: File path resolved relative to the SQL file containing the directive
|
- `path`: Relative file path beneath the SQL file’s directory; traversal, absolute paths, and symlinks are rejected
|
||||||
- `var`: Named placeholder to replace, such as `:message`
|
- `var`: Named placeholder to replace, such as `:message`
|
||||||
- `mode`: `text` embeds an escaped SQL string literal; `base64` embeds a base64 string literal
|
- `mode`: `text` embeds an escaped SQL string literal; `base64` embeds a base64 string literal
|
||||||
|
|
||||||
The directive comment is removed from the SQL, and every matching placeholder is replaced before the script is listed or executed.
|
A directive must be a standalone SQL line comment. Directive-like text inside
|
||||||
|
quoted strings, quoted identifiers, block comments, or dollar-quoted bodies is
|
||||||
|
ordinary SQL content. The directive is removed, and every matching placeholder in
|
||||||
|
executable SQL is replaced once before listing or execution. Replacements are
|
||||||
|
literal, preserving `$1`, `${2}`, `$HOME`, and other dollar text; inserted content
|
||||||
|
is never scanned for another placeholder. Duplicate embed variables or variables
|
||||||
|
used only inside strings/comments are errors. PostgreSQL casts (`::type`) remain
|
||||||
|
unchanged. Asset-call named parameters follow these same lexical boundaries;
|
||||||
|
repeated parameters share one positional argument.
|
||||||
|
|
||||||
|
Embed and manifest asset reads share root-scoped containment and a 64 MiB per-file
|
||||||
|
limit. Only regular files are accepted. The SQL directory reader uses its selected
|
||||||
|
script tree as an explicit asset root; embed paths remain relative to each script.
|
||||||
|
Symlinks in asset paths are rejected, and containment is enforced again at open time.
|
||||||
|
Manifest entries are checked at discovery and execution, so replacing a file with an
|
||||||
|
escaping symlink after scanning cannot expose an external file. No per-directive
|
||||||
|
external-file override is supported; copy required assets into the allowed tree.
|
||||||
|
|
||||||
|
Library callers can choose a trusted root and size limit with
|
||||||
|
`assetloader.ProcessEmbedDirectivesWithOptions(..., assetloader.EmbedOptions{
|
||||||
|
AssetRoot: root, MaxFileBytes: limit})`; zero means the 64 MiB default, and negative
|
||||||
|
or unbounded limits are rejected. Manifest items retain `AssetRoot` and
|
||||||
|
`MaxFileBytes` for execution-time validation.
|
||||||
|
|
||||||
|
For scripts loaded from a model rather than a SQL directory, the executor requires
|
||||||
|
trusted writer configuration `Metadata["asset_root"]` before expanding directives.
|
||||||
|
Model `source_path` metadata supplies resolution context only; it does not grant
|
||||||
|
local-file access. Missing or out-of-root source paths fail before reading assets.
|
||||||
|
|
||||||
|
|
||||||
## Commands
|
## Commands
|
||||||
|
|
||||||
|
|||||||
@@ -570,3 +570,13 @@ COMMIT;
|
|||||||
- [RelSpec Documentation](../README.md)
|
- [RelSpec Documentation](../README.md)
|
||||||
- [Model Structure Reference](../pkg/models/)
|
- [Model Structure Reference](../pkg/models/)
|
||||||
- [Example Templates](../examples/templates/)
|
- [Example Templates](../examples/templates/)
|
||||||
|
|
||||||
|
## Output path safety
|
||||||
|
|
||||||
|
In schema, domain, script, and table modes, `--output` always names a directory,
|
||||||
|
including a nonexistent directory without a trailing slash. An empty value uses
|
||||||
|
the current directory. Database mode uses a single output file as before.
|
||||||
|
Generated paths must stay relative to that directory. Traversal, absolute or
|
||||||
|
Windows-specific unsafe paths, symlinks, filename collisions (including case
|
||||||
|
variants), and file/directory collisions are rejected before publishing artifacts.
|
||||||
|
Nested patterns such as `{{.ParentSchema.Name}}/{{.Name}}.txt` remain supported.
|
||||||
|
|||||||
+359
@@ -0,0 +1,359 @@
|
|||||||
|
# Project audit 1
|
||||||
|
|
||||||
|
Date: 2026-10-12
|
||||||
|
Revision reviewed: `bfe98528265b45be7bfdcdf025710981bae06a7b`
|
||||||
|
Perspectives: software engineering, security, maintenance, and databases.
|
||||||
|
|
||||||
|
## Repair progress
|
||||||
|
|
||||||
|
Implementation batches (2026-10-12):
|
||||||
|
|
||||||
|
- [x] **A01, PostgreSQL failure propagation:** stop on the first SQL failure by default; explicit continuation still returns a structured `ExecutionFailure`. Reports include skipped entries, and requested report write/close failures are errors. Live tests cover a middle-statement failure in both modes and verify dependent jobs do not start.
|
||||||
|
- [x] **A02, implicit replacement removed:** both full-DDL and diff type changes propagate conversion errors without renaming/replacing the column. Live populated-data tests verify original values, column count/type, unique-index identity, and constraint enforcement survive incompatible conversions.
|
||||||
|
- [x] **Related defect exposed by A01:** full-DDL column additions now use native `ADD COLUMN IF NOT EXISTS`, fixing duplicate-column errors for flattened output that previously appeared successful.
|
||||||
|
- [x] **A01, shared execution results:** PostgreSQL, SQLite, SQL Server, and raw script execution now use `writers.ExecutionReport` / `ExecutionFailure`. Stop/continue modes always fail on SQL errors; counters/results survive early failure and aggregate across script schemas. Reports distinguish applied, failed, skipped, rolled-back, and uncertain operations. Generated DDL supports optional atomic execution; raw scripts report script-managed outcomes without promising statement-level commit/rollback. Requested report failures are errors in every executor. Live PostgreSQL, SQL Server, SQLite, and dependent-job regressions pass.
|
||||||
|
- [x] **A02, conversion expressions and preflight:** column metadata `pgsql_using` configures an explicit SQL conversion expression; `pgsql_preflight` adds a row predicate. Full-DDL and diff plans evaluate every existing value before schema mutations, and live preflight failure stops even with continuation enabled. Tests cover later invalid rows, preservation of a constraint scheduled for removal, custom conversions, predicate rejection, invalid configuration, and block-delimiter collisions. Standalone schema generation also includes preflights.
|
||||||
|
|
||||||
|
- [x] **A03, contained template output:** multi-file output has an explicit directory root, portable relative filename validation, and preflight collision/symlink checks. Rendering/validation complete before writing; exclusive per-file staging and root-scoped rename prevent generated filenames from escaping the root. Multi-file atomic publication remains A16 work.
|
||||||
|
- [x] **A04, contained asset reads:** embed directives and manifest execution use shared root-scoped reads, symlink checks, regular-file validation, and a bounded 64 MiB default. Manifest paths are rechecked at execution. Imported source-path metadata cannot authorize file reads without a trusted configured asset root. Library callers can explicitly configure a root/size limit; directives cannot opt out of containment.
|
||||||
|
- [x] **A05, PostgreSQL quoting:** schema/full-DDL/diff paths and embedded templates always quote model identifiers, quote literal/catalog/regclass values, sanitize names in generated line comments, and choose collision-safe dollar delimiters for procedural and audit function bodies. NUL is rejected before output; migration comparisons preserve identifier case. Raw SQL expressions remain an explicit executable-input boundary.
|
||||||
|
- [x] **Requested lowercase option:** `--lowercase` on convert/batch/merge/split/templ/job run, job `options.lowercase_identifiers`, and PostgreSQL `WriterOptions.LowercaseIdentifiers` normalize names and structured references on a copy, rejecting collisions. SQL expressions, defaults, comments, enum values, and data keep their case; callers must adjust references inside raw SQL themselves.
|
||||||
|
- [x] **A08, SQL-aware substitution/execution:** embeds use one-pass literal substitution, preserving dollar characters and preventing replacement rescanning. Named parameters and embed directives respect SQL strings, quoted identifiers, nested comments, casts, and dollar bodies; duplicate/unused embed variables fail. MySQL executes structured generated statements without splitting multiline literals or stripping their contents.
|
||||||
|
|
||||||
|
- [x] **A06, nullable MySQL catalogs:** constraint references, functional-index columns/expressions, index prefix/direction fields, column dimensions/collations/generation metadata, and optional table auto-increment counters use nullable destinations. Queries qualify joined columns. Live primary/unique/composite/functional-key reads pass, including unsigned auto-increment values beyond signed 64-bit range.
|
||||||
|
- [x] **A07, scoped MySQL fidelity and diagnostics:** native types/default spellings and typed dimensions/index parts survive MySQL → model → JSON/YAML → DDL. Output emits foreign keys/actions after all tables, checks/enforcement, unique/ordinary/functional/prefix/descending/invisible/fulltext indexes, generated columns, comments/collations/charsets, ON UPDATE, invisible columns, and auto-increment counters. Known unsupported objects/attributes produce warnings or `--strict-conversion` / job `options.strict_conversion` / library `StrictConversion` failures before output or connection. See [the scoped capability matrix](MYSQL_FIDELITY.md); this covers new-table DDL on MySQL 8.4 / InnoDB, not all engines, MariaDB versions, other format mappings, or migrations.
|
||||||
|
|
||||||
|
A06–A07 verification (2026-10-12): live MySQL 8.4.11 tests compare source/target `SHOW CREATE TABLE` definitions (normalizing temporary target suffixes and ordinary-index display order), re-read the target, and verify FK/check enforcement and binary/bit/multiline/backslash/default values. The suite covers composite/functional unique keys, reverse table dependency order, generated/check/index expressions containing SQL literals, precision, unsigned ranges, and multiple FKs to the same target. It exposed and fixed truncated binary defaults in `information_schema` by extracting native defaults from SQL-aware `SHOW CREATE TABLE` definitions. Native SHOW CREATE also avoids stale auto-increment statistics, and escaped catalog expressions are decoded before rendering. Unsupported views/triggers/partitioned tables produce warnings and strict failures; strict failure preserves existing files and happens before opening an output connection. CLI dry-run and per-job strict options are tested. The MySQL container test runner now selects reader/writer packages by default. `go test -mod=vendor -p 1 ./...` passed with `RELSPEC_TEST_MYSQL_CONN` configured; `go vet -mod=vendor ./...`, CLI strict/dry-run checks, shell syntax, and `git diff --check` passed. The MySQL container runner also passed using its schema-scoped account; its disposable server enables trigger-fixture creation after final-server readiness. PostgreSQL and SQL Server live variables were unset for this batch. Checks used `go1.27.1-X:nodwarf5 linux/amd64`; declared-minimum-toolchain coverage remains A13 work. Disposable MySQL instances were removed after verification. A08 now also has live MySQL coverage for embedded statement delimiters/default contents.
|
||||||
|
|
||||||
|
A05–A08 verification: unit regressions and live PostgreSQL 18 checks cover unusual names, comments/defaults, rerunnable file/statement output, diff migrations, sequences, primary-key replacement, audit functions/triggers, and both standard-string modes. MySQL statement preservation is unit-tested; a live MySQL instance was not used for A08. `go test -mod=vendor -p 1 ./...` passed with `RELSPEC_TEST_PG_CONN` set to the disposable instance; `go vet -mod=vendor ./...`, executable CLI help checks, and `git diff --check` passed. SQL Server tests were not configured for this batch. The disposable instance was removed after verification.
|
||||||
|
|
||||||
|
Additional observation: a parallel live full-suite run hit a PostgreSQL reader catalog race (`could not open relation with OID`) while another package's test dropped schema objects. The writer safely fell back to full DDL and its preflight still stopped all mutations, but the existing test assumed no successful read-only preflights before failure. Running live packages serially passed. Reader resilience to concurrent catalog changes remains outside A05/A08; A10 specifically concerns cancellation/timeouts.
|
||||||
|
|
||||||
|
A03–A04 verification: `go test -mod=vendor ./...`, targeted regression tests, `go vet -mod=vendor ./...`, and `git diff --check` passed. Database-dependent tests were not configured for this filesystem-only batch. Windows test binaries were cross-compiled for the template and asset packages; Windows runtime filesystem tests remain unperformed.
|
||||||
|
|
||||||
|
Verification: `go test -mod=vendor ./...` passed with `RELSPEC_TEST_PG_CONN` and `RELSPEC_TEST_MSSQL_CONN` pointing to disposable PostgreSQL 18 and SQL Server 2022 Developer instances; SQLite regressions use temporary database files. A minimal public table was seeded for the existing PostgreSQL `ReadTable` test, which requires a nonempty default schema. `go vet -mod=vendor ./...` and `git diff --check` passed. Executable CLI checks verified `scripts execute --ignore-errors`, `--continue-on-error`, and `--coe` all attempt a later script after failure and exit 1. Raw script tests intentionally make no statement-level rollback claim. A09 remains open for concurrency, history, and migration recovery; sequence/external side effects are outside transaction rollback guarantees. Original audit evidence is retained for historical context.
|
||||||
|
|
||||||
|
## Assessment
|
||||||
|
|
||||||
|
RelSpec has a useful architecture, broad format support, and substantial tests. Its most serious weaknesses are in the boundary between generating a schema and safely applying that schema to a real database. A successful process exit does not reliably mean a successful migration. Some conversions silently discard database semantics, and several file/SQL generation paths do not preserve the safety guarantees implemented elsewhere.
|
||||||
|
|
||||||
|
**Recommendation: fix the high-priority findings before relying on unattended production database writes or processing externally supplied specifications.** Continue using generated output with review and disposable database validation while these issues are addressed. This is a repository audit, not a certification of production readiness.
|
||||||
|
|
||||||
|
### Existing strengths
|
||||||
|
|
||||||
|
- Readers, writers, the intermediate model, diffing, inspection, and transformations are separated into packages.
|
||||||
|
- There are many unit and regression tests, including live PostgreSQL writer tests and SQLite tests.
|
||||||
|
- Job files have semantic validation, dependency-cycle detection, environment-based connection references, path containment checks, and symlink checks.
|
||||||
|
- Single-file job conversions already use temporary output followed by rename.
|
||||||
|
- The asset manifest executor binds file bytes as database parameters.
|
||||||
|
- The repository includes a private vulnerability reporting policy, vendored dependencies, and release-time vet/static analysis/vulnerability-check steps.
|
||||||
|
|
||||||
|
These controls should be extended and made consistent, rather than replaced wholesale.
|
||||||
|
|
||||||
|
## Scope and verification
|
||||||
|
|
||||||
|
Reviewed the CLI, job handling, database readers/writers, PostgreSQL templates and migration generation, asset embedding, generic templates, connection builder, updater, build/release workflows, documentation, and the VS Code extension. Review depth was greatest for database mutation and input/output trust boundaries; every parser and format combination was not exhaustively audited.
|
||||||
|
|
||||||
|
Checks performed:
|
||||||
|
|
||||||
|
| Check | Result |
|
||||||
|
| --- | --- |
|
||||||
|
| `GOCACHE=/tmp/relspec-audit-gocache go test -mod=vendor ./...` | Initial sandbox run failed because HTTP tests could not open loopback sockets. Approved rerun outside the sandbox passed. |
|
||||||
|
| `GOCACHE=/tmp/relspec-audit-gocache go vet -mod=vendor ./...` | Passed. |
|
||||||
|
| `make fmt-check GOLANGCI_LINT=false` | Incorrectly passed, confirming that a failed formatter can be hidden. |
|
||||||
|
| Temporary Go reproduction using exported package APIs | Confirmed template path escape, embed path escape, embed dollar corruption, invalid PostgreSQL identifier/literal rendering, omitted MySQL FK/index output, and placeholder matching inside SQL strings. |
|
||||||
|
|
||||||
|
The local compiler reported `go1.27.1-X:nodwarf5 linux/amd64`; `go.mod` requires `1.26.9`. This was not a verification under the declared minimum toolchain. No live PostgreSQL, MySQL, or SQL Server connection was configured for the audit. Tests requiring those connections skipped; a passing package does not demonstrate live database correctness. Race testing, fuzz campaigns, benchmarks, `staticcheck`, and a fresh `govulncheck` scan were not performed. No claims about specific dependency CVEs are made.
|
||||||
|
|
||||||
|
Reproduction files and logs were placed in `/tmp`; the only intended repository change is this document. No real credentials or user data were used in reproductions. Database injection and migration effects were assessed from code/generated SQL, not executed against a database.
|
||||||
|
|
||||||
|
### Priority and evidence notation
|
||||||
|
|
||||||
|
- **P1 / High:** address before unattended database use or handling less-trusted inputs; credible data-integrity, confidentiality, or integrity impact.
|
||||||
|
- **P2 / Medium:** schedule in the next maintenance cycle; operational reliability, reproducibility, or defense-in-depth issue.
|
||||||
|
- **P3 / Low:** documentation or developer-experience issue.
|
||||||
|
- **Reproduced:** a local executable check demonstrated the stated behavior.
|
||||||
|
- **Code-confirmed:** the implementation establishes the behavior, but the complete external-system scenario was not executed.
|
||||||
|
|
||||||
|
## Findings
|
||||||
|
|
||||||
|
### A01 — PostgreSQL writes report success after statement failures
|
||||||
|
|
||||||
|
**Repair status: addressed.** The shared executor now enforces stop/continue behavior and nonzero failure across PostgreSQL, SQLite, SQL Server, and raw scripts, including requested report errors. Reports include individual outcomes, skipped operations, confirmed rollbacks, and uncertain finalization. Atomic generated-DDL runs are optional; raw scripts explicitly report script-managed execution. A09 remains a separate migration-management/concurrency contract.
|
||||||
|
|
||||||
|
**P1 / High · Engineering, database · Code-confirmed**
|
||||||
|
|
||||||
|
Evidence: `pkg/writers/pgsql/writer.go:2200–2322`, especially `executeStatements` and `finishReport`; `pkg/writers/pgsql/live_execute_test.go:144` explicitly tests continuing after failure.
|
||||||
|
|
||||||
|
Every failed statement is recorded and execution continues. `finishReport` prints an error count but returns `nil`. Failure to write the requested report is also only a warning. This behavior is unconditional in this execution path. A deployment or dependent job can proceed after an incomplete schema change because the writer returned success.
|
||||||
|
|
||||||
|
**Fix:** return a structured error when any statement fails; stop on the first error by default. A continue-on-error option should control continuation, separately from the final exit status. Aggregate results consistently across PostgreSQL, SQLite, SQL Server, and script execution. Treat failure to produce an explicitly requested execution report as an error.
|
||||||
|
|
||||||
|
**Acceptance:** force a middle statement to fail; the CLI exits nonzero, dependent jobs do not run, and the report distinguishes applied, failed, skipped, and rolled-back operations. Test both stop and continue modes.
|
||||||
|
|
||||||
|
### A02 — Failed type conversion silently substitutes an empty column
|
||||||
|
|
||||||
|
**Repair status: addressed.** The shared type-change template propagates conversion errors. Configurable `pgsql_using` expressions and `pgsql_preflight` predicates are supported, with automatic whole-column cast evaluation before schema mutations. Full-DDL and diff live tests verify preservation of populated columns and unique dependencies, custom conversions, and rejected predicates. Preflight is advisory under concurrent writes; atomic generated-DDL execution is available for transactionally supported changes, and separately reviewed rename/replacement remains required.
|
||||||
|
|
||||||
|
**P1 / High · Database · Code-confirmed**
|
||||||
|
|
||||||
|
Evidence: `pkg/writers/pgsql/templates/alter_column_type_with_check.tmpl:18–27`; `pkg/writers/pgsql/migration_writer.go:491`; `pkg/writers/pgsql/writer.go:514`.
|
||||||
|
|
||||||
|
The type-change block catches `WHEN OTHERS`, renames the existing column, and adds a fresh column with the original name. For example, converting populated text values containing non-numeric data to an integer can fail the cast and activate this fallback. The application then sees an empty replacement column. Existing indexes, constraints, and dependencies may remain attached to the renamed column. Original values are not necessarily deleted, but the application's expected column no longer contains them.
|
||||||
|
|
||||||
|
The broad exception handler also treats unrelated failures as permission to attempt this schema rewrite.
|
||||||
|
|
||||||
|
**Fix:** fail on conversion errors. Require a separately reviewed migration for any rename-and-replace operation, including backfill, validation, dependency handling, and recovery. Support an explicit conversion expression and preflight checks where appropriate.
|
||||||
|
|
||||||
|
**Acceptance:** migrating populated incompatible data leaves the original column, values, and dependencies intact and returns an actionable failure. No rename/replacement occurs implicitly.
|
||||||
|
|
||||||
|
### A03 — Generated template filenames escape the output directory
|
||||||
|
|
||||||
|
**Repair status: addressed.** Multi-file roots, generated paths, collisions, symlinks, and root-scoped writes are covered by regression tests. A nonexistent output path now consistently denotes the directory itself rather than its parent.
|
||||||
|
|
||||||
|
**P1 / High · Security · Reproduced**
|
||||||
|
|
||||||
|
Evidence: `pkg/writers/template/writer.go:249–308`.
|
||||||
|
|
||||||
|
The default filename pattern uses `{{.Name}}.txt`. `generateFilename` joins the rendered name with the destination without checking containment; `writeOutput` creates directories and writes the resulting path. A schema named `../escaped` wrote `escaped.txt` beside the selected output directory in the reproduction.
|
||||||
|
|
||||||
|
An imported model can therefore cause writes outside the intended destination, including overwriting files writable by the current user. Job-level checks validate the declared output path, not every subsequently generated filename. Exploitation requires the user to process the supplied model in a multi-file template mode.
|
||||||
|
|
||||||
|
**Fix:** validate every rendered filename against a clearly defined output root; reject traversal, unsafe absolute paths, and symlink escapes. Detect filename collisions before writing. Prefer root-scoped filesystem operations so validation and opening cannot be separated by a symlink replacement.
|
||||||
|
|
||||||
|
**Acceptance:** malicious names, filename patterns, symlink parents, duplicate names, and platform-specific paths cannot write outside the root or silently overwrite another generated artifact.
|
||||||
|
|
||||||
|
### A04 — SQL embed directives can read outside the script directory
|
||||||
|
|
||||||
|
**Repair status: addressed.** Shared bounded root-scoped reads protect directives and manifest assets, including revalidation after discovery. Model source-path metadata alone no longer grants local file access.
|
||||||
|
|
||||||
|
**P1 / High · Security · Reproduced**
|
||||||
|
|
||||||
|
Evidence: `pkg/assetloader/embed.go:73–74`; compare the containment checks in `pkg/assetloader/loader.go` and `pkg/jobs/jobs.go`.
|
||||||
|
|
||||||
|
Embed paths are cleaned and joined to the script directory, then passed directly to `os.ReadFile`. A directive using `path=../synthetic-secret.txt` successfully embedded a sibling file. There is no equivalent root or symlink policy for this path. Script metadata also supplies the source path used for resolution.
|
||||||
|
|
||||||
|
A supplied script can cause local readable files to be copied into generated SQL or sent to the destination database. Permission to execute SQL against a database should not implicitly grant unrestricted local-file access. This is a local input trust-boundary issue, not an unauthenticated network endpoint.
|
||||||
|
|
||||||
|
**Fix:** define an explicit asset root, contain all embed reads within it, check symlink resolution, limit file size, and document any opt-in allowance for external files. Reuse the policy across manifest assets and embed directives.
|
||||||
|
|
||||||
|
**Acceptance:** traversal and symlink escape fail before reading; valid in-root text and binary assets still work; diagnostics do not print file contents.
|
||||||
|
|
||||||
|
### A05 — PostgreSQL SQL quoting is incomplete across syntactic contexts
|
||||||
|
|
||||||
|
**P1 / High · Security, database · Reproduced rendering defects**
|
||||||
|
|
||||||
|
Evidence: `pkg/writers/pgsql/template_functions.go:134`; `pkg/writers/pgsql/templates/add_column_with_check.tmpl:5–7`; `create_foreign_key_with_check.tmpl`; `alter_column_type_with_check.tmpl`.
|
||||||
|
|
||||||
|
Two separate defects were confirmed:
|
||||||
|
|
||||||
|
1. `quoteIdent` does not actually check reserved keywords, despite its comment. The template executor generated `CREATE TABLE IF NOT EXISTS public.order (select text);` for legal names requiring quoting.
|
||||||
|
2. Catalog lookups insert names directly into single-quoted SQL strings. A table named `O'Reilly` generated `table_name = 'O'Reilly'`. Quoting a name for an identifier elsewhere does not escape it as a string literal here.
|
||||||
|
|
||||||
|
Fixed `DO $$ ... $$` delimiters create another context that must be protected from embedded delimiter text. Besides breaking valid database names, insufficient context-aware quoting is an injection risk when less-trusted metadata is rendered and executed using privileged database credentials. A complete injection exploit was not executed during this audit. Raw SQL expressions are a separate, intentional executable-input boundary and should be documented as such.
|
||||||
|
|
||||||
|
**Fix:** centralize identifier, literal, qualified-name, and block-delimiter handling. Always quote identifiers where practical; parameterize values where supported. Generate collision-safe dollar delimiters or avoid interpolating arbitrary content into fixed blocks. Apply this across all templates, not only the examples above.
|
||||||
|
|
||||||
|
**Acceptance:** generated SQL executes correctly for reserved words, apostrophes, double quotes, Unicode, newlines, and dollar delimiters, and cannot introduce additional SQL statements through a name.
|
||||||
|
|
||||||
|
PostgreSQL documents the distinct identifier, string, and dollar-quote rules in [Lexical Structure](https://www.postgresql.org/docs/current/sql-syntax-lexical.html).
|
||||||
|
|
||||||
|
### A06 — MySQL constraint reading scans nullable metadata into strings
|
||||||
|
|
||||||
|
**Repair status: addressed.** Nullable metadata scans and qualified queries are covered by live MySQL 8.4 round trips with primary, unique, composite FK, and functional-index definitions.
|
||||||
|
|
||||||
|
**P1 / High · Database · Code-confirmed; live MySQL not exercised**
|
||||||
|
|
||||||
|
Evidence: `pkg/readers/mysql/reader.go:181–220`, specifically the `rs`, `rt`, and `rc` string variables passed to `rows.Scan`.
|
||||||
|
|
||||||
|
The constraint query reads `REFERENCED_TABLE_SCHEMA`, `REFERENCED_TABLE_NAME`, and `REFERENCED_COLUMN_NAME` for primary and unique constraints as well as foreign keys. Non-foreign-key rows have no referenced object and can contain SQL NULL. Scanning these into Go strings fails, aborting inspection of ordinary tables with a primary key.
|
||||||
|
|
||||||
|
**Fix:** use nullable destinations such as `sql.NullString`, or explicit `COALESCE` where an empty string has the intended meaning. Review other nullable metadata, including expression-index column names, in the same reader.
|
||||||
|
|
||||||
|
**Acceptance:** live MySQL tests read tables with primary, unique, composite foreign-key, and expression-index definitions without scan failures and preserve their metadata. The current reader tests only cover type mapping and a missing connection string.
|
||||||
|
|
||||||
|
Reference: [MySQL KEY_COLUMN_USAGE documentation](https://dev.mysql.com/doc/refman/8.0/en/information-schema-key-column-usage-table.html) and [MySQL's documented non-FK NULL behavior](https://bugs.mysql.com/bug.php?id=9587).
|
||||||
|
|
||||||
|
### A07 — MySQL conversion silently loses constraints, indexes, and type detail
|
||||||
|
|
||||||
|
**Repair status: addressed within the [documented MySQL table-schema contract](MYSQL_FIDELITY.md).** Supported features survive live round trips through JSON/YAML; known unsupported modeled/discovered features produce diagnostics or strict conversion failures. MySQL output remains creation DDL for existing databases, with engine/version/SQL-mode limits stated in the capability matrix.
|
||||||
|
|
||||||
|
**P1 / High · Database · Reproduced omissions; additional losses code-confirmed**
|
||||||
|
|
||||||
|
Evidence: `pkg/writers/mysql/writer.go:96–163`; `pkg/readers/mysql/reader.go:153–178`; `pkg/mariadb/datatypes.go`.
|
||||||
|
|
||||||
|
The writer emits columns, primary keys, and unique constraints, but does not emit foreign keys, checks, or ordinary indexes from the model. A reproduction containing both a foreign key and an index returned success while emitting neither. The reader maps `COLUMN_TYPE` to a canonical base type without populating the separate length/precision/scale fields; its tests explicitly map `bigint unsigned` to `int64`.
|
||||||
|
|
||||||
|
Consequences include lost referential integrity, missing performance-critical indexes, changed ranges, and changed string/decimal definitions. Advertising a format as supported does not communicate these losses.
|
||||||
|
|
||||||
|
**Fix:** preserve and emit supported semantics; return explicit diagnostics for every unsupported object or attribute. Add a strict conversion mode that refuses lossy output and a per-format capability matrix. Preserve unsignedness, dimensions, generated expressions, and FK actions through the intermediate representation.
|
||||||
|
|
||||||
|
**Acceptance:** inspect → model → DDL → disposable database → inspect tests compare semantic equivalence for representative MySQL schemas. Unsupported features produce visible, testable failures or warnings, never silent success.
|
||||||
|
|
||||||
|
### A08 — Embed substitution corrupts dollar characters; placeholder parsing ignores SQL syntax
|
||||||
|
|
||||||
|
**P2 / Medium · Engineering, database · Reproduced**
|
||||||
|
|
||||||
|
Evidence: `pkg/assetloader/embed.go:159–160`; `pkg/assetloader/executor.go:32–87`.
|
||||||
|
|
||||||
|
`replaceEmbedPlaceholder` passes file content into the replacement argument of `regexp.ReplaceAllString`. Dollar characters are interpreted as capture references. Embedding `price $1 ${2} $HOME $$` produced `price ; $` in the local reproduction, rather than preserving the input.
|
||||||
|
|
||||||
|
Separately, `BuildQuery` matches placeholders inside quoted strings and comments. `SELECT ':literal', :bytes` fails with an unknown-placeholder error even though `:literal` is SQL string content. Similar text-based processing makes the MySQL executor's `strings.Split(..., ";\n")` fragile for multiline defaults containing that sequence.
|
||||||
|
|
||||||
|
**Fix:** use literal replacement callbacks or explicit span assembly for embed content. Parse SQL lexical contexts for named placeholders. Execute a structured list of generated statements instead of splitting SQL text on delimiters.
|
||||||
|
|
||||||
|
**Acceptance:** dollar text, repeated placeholders, quoted strings, comments, casts, dollar-quoted bodies, and embedded statement delimiters preserve exact intended semantics.
|
||||||
|
|
||||||
|
### A09 — Migration execution lacks a coherent atomicity and concurrency contract
|
||||||
|
|
||||||
|
**P1 / High · Database, engineering · Code-confirmed**
|
||||||
|
|
||||||
|
Evidence: `pkg/writers/pgsql/writer.go:2200–2300`; `pkg/writers/sqlexec/writer.go:107–222`; `pkg/writers/sqlite/writer.go:118–164`; `pkg/writers/mysql/writer.go:165–188`.
|
||||||
|
|
||||||
|
Executors issue independent statements/scripts without a transaction spanning the operation, migration-wide lock, or applied-script ledger in these paths. A failure can leave earlier work committed. Re-running script execution repeats supplied scripts; concurrent invocations can interleave. A script may contain its own transaction, but that does not establish a consistent executor-level guarantee.
|
||||||
|
|
||||||
|
The necessary behavior differs by database and statement type: wrapping all MySQL DDL in a transaction would not provide the same rollback guarantee as transactional PostgreSQL DDL.
|
||||||
|
|
||||||
|
**Fix:** define explicit execution modes with transaction boundaries, concurrency control, retry/recovery semantics, and accurate partial-application reports. If `scripts execute` remains a raw runner, document that clearly. If it is intended as a migration manager, add migration identity, checksums, applied history, and duplicate/concurrent-run protection.
|
||||||
|
|
||||||
|
**Acceptance:** failure injection and simultaneous-run tests establish predictable behavior; supported transactional runs roll back; nontransactional runs report exactly what committed and how to resume safely.
|
||||||
|
|
||||||
|
### A10 — Database operations cannot consistently inherit cancellation or deadlines
|
||||||
|
|
||||||
|
**P2 / Medium · Engineering, operations · Code-confirmed**
|
||||||
|
|
||||||
|
Evidence: `pkg/readers/reader.go`; `pkg/writers/writer.go`; `pkg/writers/pgsql/writer.go:2205`; `pkg/writers/sqlexec/writer.go`; `pkg/readers/mysql/reader.go:25`; `pkg/writers/mysql/writer.go:184`.
|
||||||
|
|
||||||
|
Reader/writer interfaces do not accept a caller context, and several implementations start with `context.Background()`. This disconnects CLI/job cancellation from connection establishment, catalog reads, and SQL execution. A blocked statement or lock wait has no consistent application-level deadline.
|
||||||
|
|
||||||
|
**Fix:** carry caller contexts through reader/writer operations; configure connection, statement, and lock-wait limits separately. Use a signal-aware CLI root context and preserve cancellation errors in job results.
|
||||||
|
|
||||||
|
**Acceptance:** a blocked database query terminates promptly on cancellation or timeout, cleanup is bounded, and the job exits nonzero without starting dependents.
|
||||||
|
|
||||||
|
### A11 — Windows updater executes downloads without artifact authentication
|
||||||
|
|
||||||
|
**P2 / Medium · Security, supply chain · Code-confirmed**
|
||||||
|
|
||||||
|
Evidence: `cmd/relspec/update.go:115–152`; `.gitea/workflows/release.yml`.
|
||||||
|
|
||||||
|
After confirmation or `--yes`, the updater downloads the asset URL from release metadata and starts the resulting executable. There is no signature verification, trusted-publisher check, content-size limit, or asset-origin/redirect restriction in this path. The default release API uses HTTPS, which is a useful transport control, but it is not independent artifact authentication. Compromised release metadata or publication infrastructure could substitute an installer.
|
||||||
|
|
||||||
|
**Fix:** sign installers or release manifests and verify against an independently trusted identity before launch. Require HTTPS throughout redirects, apply a deliberate origin policy, cap downloads, and clean up failed downloads. A checksum fetched from the same compromised endpoint alone is insufficient authentication.
|
||||||
|
|
||||||
|
**Acceptance:** tampered, unsigned, wrong-publisher, oversized, and disallowed-origin assets are rejected before execution.
|
||||||
|
|
||||||
|
### A12 — Connection and development-service defaults permit avoidable exposure
|
||||||
|
|
||||||
|
**P2 / Medium · Security · Code-confirmed**
|
||||||
|
|
||||||
|
Evidence: `pkg/ui/connstring.go:42–49`; `docker-compose.yml`; `Makefile`'s `docker-up` target.
|
||||||
|
|
||||||
|
The UI starts PostgreSQL and SQL Server connection fields with encryption disabled; changing the host does not change that setting. Compose publishes database ports without a loopback host restriction and supplies fixed development passwords. These are development credentials, not evidence of leaked production secrets, but the services can become reachable on host network interfaces.
|
||||||
|
|
||||||
|
**Fix:** default remote connections to authenticated encrypted transport and make local exceptions explicit. Bind development database ports to `127.0.0.1` by default; document how to deliberately expose a test service. Prefer a limited inspection role over administrative defaults.
|
||||||
|
|
||||||
|
**Acceptance:** changing a connection to a remote host does not silently retain an insecure default; standard local test startup only exposes loopback ports.
|
||||||
|
|
||||||
|
### A13 — CI does not consistently test the declared toolchain or live writer paths
|
||||||
|
|
||||||
|
**P1 / High · Maintenance, database assurance · Code-confirmed configuration defects**
|
||||||
|
|
||||||
|
Evidence: `.github/workflows/ci.yml:15,58,79`; `.github/workflows/integration-tests.yml:59–76`; `.gitea/workflows/release.yml:48–49`; `Makefile:92–108`; `tests/dbtest/dbs/mysql.sh`.
|
||||||
|
|
||||||
|
- GitHub CI specifies Go 1.24/1.25 while `go.mod` requires 1.26.9. Automatic toolchain switching can make that matrix misleading; with switching disabled those jobs cannot build the module as declared.
|
||||||
|
- The GitHub integration job runs directly on the host but connects to `postgres`, the service-container name, rather than the published loopback port. GitHub documents `localhost` for this job layout. This networking conclusion applies to GitHub-hosted execution; the Gitea runner configuration was not inspected.
|
||||||
|
- `make test-integration` runs the PostgreSQL reader and `tests/integration`, but omits live tests under `pkg/writers/pgsql` and `pkg/writers/sqlexec`. Release `go test ./...` has no configured database service/connection, so those tests can skip there too.
|
||||||
|
- MySQL has container scaffolding but no Go tests consuming `RELSPEC_TEST_MYSQL_CONN`; this leaves A06 undetected.
|
||||||
|
- `make test-unit` excludes the entire PostgreSQL reader package, including `queries_pure_test.go`.
|
||||||
|
|
||||||
|
**Fix:** use `go-version-file: go.mod`, explicitly test supported toolchains, correct GitHub service networking, and run all live readers and writers against disposable databases in required CI jobs. Make unexpected integration-test skips fail the integration job. Verify which workflows the actual Gitea deployment discovers; CI badges alone do not establish execution.
|
||||||
|
|
||||||
|
**Acceptance:** CI artifacts show real reader/writer execution, not just green packages with skipped live tests, and demonstrate MySQL constraint inspection and failing-migration behavior.
|
||||||
|
|
||||||
|
Reference: [GitHub PostgreSQL service-container networking](https://docs.github.com/en/actions/tutorials/use-containerized-services/create-postgresql-service-containers).
|
||||||
|
|
||||||
|
### A14 — Formatting gate ignores formatter failure
|
||||||
|
|
||||||
|
**P2 / Medium · Maintenance · Reproduced**
|
||||||
|
|
||||||
|
Evidence: `Makefile:62–66`.
|
||||||
|
|
||||||
|
The recipe captures formatter stdout and checks whether it is empty, without preserving the command's failure status. `make fmt-check GOLANGCI_LINT=false` prints `Checking formatting...` and exits successfully. A missing/broken formatter or dependency/toolchain failure can therefore pass this gate without checking formatting.
|
||||||
|
|
||||||
|
**Fix:** explicitly fail when the formatter command fails before inspecting its diff output. Pin the formatter version so the gate is reproducible.
|
||||||
|
|
||||||
|
**Acceptance:** a formatter that exits nonzero with empty stdout fails the target; clean formatting passes; a real diff fails.
|
||||||
|
|
||||||
|
### A15 — Release artifacts can disagree with the requested release tag
|
||||||
|
|
||||||
|
**P2 / Medium · Maintenance, supply chain · Code-confirmed**
|
||||||
|
|
||||||
|
Evidence: `.gitea/workflows/release.yml:55–94,241–269,297–314`; upload steps around lines 113–130, 278–289, and 360–371; `Makefile:84,143,256,273`.
|
||||||
|
|
||||||
|
For manual dispatch with a tag input, checkout does not explicitly select that tag. Binaries and packages are built from the dispatched checkout, while the source archives use the supplied tag and version strings claim that version. These can represent different commits. Uploads use `curl -s` without failing on HTTP error status, so publication failures can also look successful.
|
||||||
|
|
||||||
|
Additional reproducibility gaps: ordinary `make build` runs `go mod tidy` through `deps`; analysis tools use `@latest`; `release-version` lacks test/build prerequisites; `rerelease` force-moves an existing release tag.
|
||||||
|
|
||||||
|
**Fix:** resolve and validate the release tag once, check out its commit in every artifact job, and record that commit in a manifest. Fail on HTTP errors and verify the uploaded inventory. Keep normal builds read-only with respect to dependency manifests, pin build tools, and publish immutable versions with consistent quality gates.
|
||||||
|
|
||||||
|
**Acceptance:** dispatching a historical tag from a newer branch produces binaries and source from the same historical commit; failed uploads fail the workflow; normal builds leave the working tree unchanged.
|
||||||
|
|
||||||
|
### A16 — Output replacement guarantees differ across commands and writers
|
||||||
|
|
||||||
|
**P2 / Medium · Engineering, maintenance · Code-confirmed**
|
||||||
|
|
||||||
|
Evidence: `cmd/relspec/job.go:859–886`; `pkg/writers/template/writer.go:288–308`; `pkg/writers/mysql/writer.go:54–68`; `pkg/writers/json/writer.go:56–58`.
|
||||||
|
|
||||||
|
Job single-file conversion has an atomic-write wrapper, but direct writers and template output write or truncate final paths in place. Some writer cleanup functions ignore close errors. A disk error, failed generation, or interrupted multi-file run can replace a previously valid artifact with incomplete output or leave a mixed generation.
|
||||||
|
|
||||||
|
**Fix:** centralize output staging and finalization for direct CLI use and jobs. Handle close/flush errors, use exclusive unpredictable temporary files, preflight filename collisions, and define a staging/manifest strategy for multi-file generations. Preserve existing files when generation fails.
|
||||||
|
|
||||||
|
**Acceptance:** injected generation/write/finalization failures leave the previous valid output intact and return an error; no command claims success with a partial replacement.
|
||||||
|
|
||||||
|
## Perspective-specific maintenance needs
|
||||||
|
|
||||||
|
These are follow-up investments and scope gaps, not additional proven vulnerabilities.
|
||||||
|
|
||||||
|
### Software engineering
|
||||||
|
|
||||||
|
- **A common capability registry:** format construction is repeated across `convert.go`, `diff.go`, `merge.go`, `inspect.go`, and `edit.go`. Job format validation is another list. Centralize reader/writer creation and capabilities so adding a format does not require synchronized switches throughout the CLI.
|
||||||
|
- **Typed execution options and results:** connection settings, execution counters, and flags are carried through string-keyed metadata maps. Introduce typed configuration/result structures for live execution while retaining metadata for extensions.
|
||||||
|
- **Model validation before generation:** JSON/YAML readers primarily unmarshal and return models. Establish a shared normalization/validation boundary for nil entries, duplicate objects, unresolved references, incompatible attributes, and absent required fields. The inspector is useful but should not be the only optional path to structural safety.
|
||||||
|
- **Smaller database writer components:** `pkg/writers/pgsql/writer.go` is 2,643 lines and combines generation, execution, reporting, type/index policy, and statement parsing. Separate these responsibilities to make security and migration changes easier to review.
|
||||||
|
- **Adversarial parser tests:** no `Fuzz...` tests were found in `pkg` or `cmd`. Add fuzz targets for DBML/directives, ORM readers, embed handling, SQL lexical processing, and invalid intermediate models. Add large-schema benchmarks before introducing concurrency or caching.
|
||||||
|
|
||||||
|
### Security
|
||||||
|
|
||||||
|
- Document the trust model: SQL scripts, expressions/defaults, asset calls, schema metadata, templates, and release assets have different execution and local-file capabilities.
|
||||||
|
- Test redaction across every CLI/library logging path, not just the job logger and masked UI preview. Cover URL-encoded passwords, DSN parse errors, and database error messages without printing real credentials in tests.
|
||||||
|
- Run pinned vulnerability checks in required change-validation workflows and record the tool/database versions. Produce release checksums and an SBOM; signing and verification are covered by A11.
|
||||||
|
- Use scoped database roles: inspection should not require DDL rights, and schema application should not normally use a server administrator account.
|
||||||
|
|
||||||
|
### Maintenance
|
||||||
|
|
||||||
|
- Update contradictory documentation. README advertises Go 1.24.0; CONTRIBUTING says 1.25+; `go.mod` requires 1.26.9. TODO says MySQL, watch mode, type mappings, Windows installer, and installer CI are missing even though implementations exist. `tests/dbtest/README.md` still says there is no MySQL driver/reader.
|
||||||
|
- Document actual feature completeness, not just format names. Distinguish implemented, partial, experimental, and unsupported behavior with links to tests.
|
||||||
|
- The VS Code extension's `renderTemplate` is a placeholder and syntax validation mainly counts delimiters per line (`vscode-extension/relspec-template-editor/src/extension.ts`). Label it experimental or connect it to the actual Go parser/renderer. Reuse and dispose a single diagnostics collection rather than creating one on each validation.
|
||||||
|
- Align GitHub and Gitea workflow ownership, tool versions, required checks, release policies, and developer setup. Add Windows execution tests for filesystem behavior and installer/updater code, rather than relying solely on cross-compilation.
|
||||||
|
|
||||||
|
### Database specialization
|
||||||
|
|
||||||
|
- Publish an object/attribute fidelity matrix covering every supported source/destination, including views, sequences, enums, generated/identity columns, FK actions/deferral, expression/partial indexes, precision/scale, unsigned types, and permissions.
|
||||||
|
- PostgreSQL generation needs explicit fidelity review beyond tables: the model includes views, enums, permissions, and standalone sequences, but the main PostgreSQL writer does not provide corresponding complete emission paths. Its sequence generation focuses on table primary keys; scanning views to infer extensions does not emit the views. Unsupported objects need warnings or rejection.
|
||||||
|
- Establish migration safety policy: destructive-change approval in the product, reviewed plans, populated-data preflight, lock budgets, backfills, recovery, and post-apply verification. A dry run that only validates options is not proof that a migration can execute safely.
|
||||||
|
- Test semantic convergence: after applying a model, re-inspect and diff; a second application should generate no unintended change. Include populated tables, cross-schema dependencies, generated columns, reserved names, and failure recovery.
|
||||||
|
- Keep database-specific transaction and DDL behavior explicit. Do not promise uniform rollback or schema support across PostgreSQL, SQLite, SQL Server, and MySQL.
|
||||||
|
|
||||||
|
## Prioritized repair plan
|
||||||
|
|
||||||
|
| Order | Work | Completion evidence |
|
||||||
|
| --- | --- | --- |
|
||||||
|
| 1 | Fix misleading success and unsafe type fallback: A01–A02 | Failure-injection tests prove nonzero exit and preservation of existing data/schema. |
|
||||||
|
| 2 | Close file and SQL rendering boundaries: A03–A05, A08 | Adversarial filename, embed, identifier, literal, and delimiter cases pass. |
|
||||||
|
| 3 | Repair MySQL reading and fidelity: A06–A07 | Live round trips preserve constraints, indexes, and type dimensions; unsupported features fail explicitly. |
|
||||||
|
| 4 | Restore trustworthy CI coverage: A13–A14 | All relevant live writers run, toolchain matches policy, and tool failures fail gates. |
|
||||||
|
| 5 | Define execution safety: A09–A10 | Documented transaction/concurrency modes, cancellation, recovery, and precise results. |
|
||||||
|
| 6 | Harden distribution and output: A11–A12, A15–A16 | Verified installer artifacts, consistent release commit, safer defaults, and staged file replacement. |
|
||||||
|
| 7 | Reduce maintenance debt | Capability registry, typed options, accurate docs, supported-feature matrix, fuzzing, and measured performance baseline. |
|
||||||
|
|
||||||
|
Avoid expanding the format list or adding a web/API layer until the existing execution and fidelity contracts are dependable. The current code already offers enough functionality that correctness, safety, and maintainability will yield more value than additional breadth.
|
||||||
@@ -0,0 +1,24 @@
|
|||||||
|
# Maintainer: Hein (Warky Devs) <hein@warky.dev>
|
||||||
|
pkgname=relspec-bin
|
||||||
|
pkgver=1.0.91
|
||||||
|
pkgrel=1
|
||||||
|
pkgdesc="RelSpec is a comprehensive database relations management tool that reads, transforms, and writes database table specifications across multiple formats and ORMs."
|
||||||
|
arch=('x86_64' 'aarch64')
|
||||||
|
url="https://git.warky.dev/wdevs/relspecgo"
|
||||||
|
license=('MIT')
|
||||||
|
provides=('relspec')
|
||||||
|
conflicts=('relspec')
|
||||||
|
options=('!debug')
|
||||||
|
source_x86_64=("$pkgname-$pkgver-x86_64.tar.gz::$url/releases/download/v$pkgver/relspec_${pkgver}_linux_amd64.tar.gz")
|
||||||
|
source_aarch64=("$pkgname-$pkgver-aarch64.tar.gz::$url/releases/download/v$pkgver/relspec_${pkgver}_linux_arm64.tar.gz")
|
||||||
|
sha256sums_x86_64=('SKIP')
|
||||||
|
sha256sums_aarch64=('SKIP')
|
||||||
|
|
||||||
|
package() {
|
||||||
|
# Binary
|
||||||
|
install -Dm755 "relspec" "$pkgdir/usr/bin/relspec"
|
||||||
|
install -Dm644 "LICENSE" "$pkgdir/usr/share/licenses/$pkgname/LICENSE"
|
||||||
|
|
||||||
|
# Default config dir
|
||||||
|
install -dm755 "$pkgdir/etc/relspec"
|
||||||
|
}
|
||||||
@@ -1,6 +1,6 @@
|
|||||||
# Maintainer: Hein (Warky Devs) <hein@warky.dev>
|
# Maintainer: Hein (Warky Devs) <hein@warky.dev>
|
||||||
pkgname=relspec
|
pkgname=relspec
|
||||||
pkgver=1.0.89
|
pkgver=1.0.91
|
||||||
pkgrel=1
|
pkgrel=1
|
||||||
pkgdesc="RelSpec is a comprehensive database relations management tool that reads, transforms, and writes database table specifications across multiple formats and ORMs."
|
pkgdesc="RelSpec is a comprehensive database relations management tool that reads, transforms, and writes database table specifications across multiple formats and ORMs."
|
||||||
arch=('x86_64' 'aarch64')
|
arch=('x86_64' 'aarch64')
|
||||||
@@ -1,5 +1,5 @@
|
|||||||
Name: relspec
|
Name: relspec
|
||||||
Version: 1.0.89
|
Version: 1.0.91
|
||||||
Release: 1%{?dist}
|
Release: 1%{?dist}
|
||||||
Summary: RelSpec is a comprehensive database relations management tool that reads, transforms, and writes database table specifications across multiple formats and ORMs.
|
Summary: RelSpec is a comprehensive database relations management tool that reads, transforms, and writes database table specifications across multiple formats and ORMs.
|
||||||
|
|
||||||
|
|||||||
+93
-18
@@ -9,10 +9,22 @@ import (
|
|||||||
"strconv"
|
"strconv"
|
||||||
"strings"
|
"strings"
|
||||||
"unicode/utf8"
|
"unicode/utf8"
|
||||||
|
|
||||||
|
"git.warky.dev/wdevs/relspecgo/pkg/safefile"
|
||||||
|
"git.warky.dev/wdevs/relspecgo/pkg/sqllex"
|
||||||
)
|
)
|
||||||
|
|
||||||
const ScriptSourcePathMetadataKey = "source_path"
|
const ScriptSourcePathMetadataKey = "source_path"
|
||||||
|
|
||||||
|
const DefaultMaxAssetBytes int64 = 64 << 20 // 64 MiB per manifest/embedded asset
|
||||||
|
|
||||||
|
// EmbedOptions are trusted caller configuration, not script/model metadata.
|
||||||
|
// An empty AssetRoot defaults to the SQL file's directory; external reads are rejected.
|
||||||
|
type EmbedOptions struct {
|
||||||
|
AssetRoot string
|
||||||
|
MaxFileBytes int64
|
||||||
|
}
|
||||||
|
|
||||||
var (
|
var (
|
||||||
embedDirectivePattern = regexp.MustCompile(`(?m)^\s*--\s*@embed:\s*(.+?)\s*$`)
|
embedDirectivePattern = regexp.MustCompile(`(?m)^\s*--\s*@embed:\s*(.+?)\s*$`)
|
||||||
embedAttrPattern = regexp.MustCompile(`([a-zA-Z_][a-zA-Z0-9_]*)=("[^"]*"|'[^']*'|\S+)`)
|
embedAttrPattern = regexp.MustCompile(`([a-zA-Z_][a-zA-Z0-9_]*)=("[^"]*"|'[^']*'|\S+)`)
|
||||||
@@ -27,7 +39,28 @@ var (
|
|||||||
// string literal. Base64 mode embeds a quoted base64 literal suitable for
|
// string literal. Base64 mode embeds a quoted base64 literal suitable for
|
||||||
// decode(:var, 'base64').
|
// decode(:var, 'base64').
|
||||||
func ProcessEmbedDirectives(sqlPath, sql string) (string, error) {
|
func ProcessEmbedDirectives(sqlPath, sql string) (string, error) {
|
||||||
directives := embedDirectivePattern.FindAllStringSubmatch(sql, -1)
|
return ProcessEmbedDirectivesWithOptions(sqlPath, sql, EmbedOptions{})
|
||||||
|
}
|
||||||
|
|
||||||
|
func embedDirectives(sql string) (result string, directives []string) {
|
||||||
|
// Only standalone line comments are directives, and only in SQL code.
|
||||||
|
result = sqllex.Walk(sql, nil, func(comment string) string {
|
||||||
|
if match := embedDirectivePattern.FindStringSubmatch(comment); match != nil {
|
||||||
|
directives = append(directives, match[1])
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
return comment
|
||||||
|
})
|
||||||
|
return result, directives
|
||||||
|
}
|
||||||
|
|
||||||
|
func HasEmbedDirectives(sql string) bool {
|
||||||
|
_, directives := embedDirectives(sql)
|
||||||
|
return len(directives) > 0
|
||||||
|
}
|
||||||
|
|
||||||
|
func ProcessEmbedDirectivesWithOptions(sqlPath, sql string, options EmbedOptions) (string, error) {
|
||||||
|
result, directives := embedDirectives(sql)
|
||||||
if len(directives) == 0 {
|
if len(directives) == 0 {
|
||||||
return sql, nil
|
return sql, nil
|
||||||
}
|
}
|
||||||
@@ -36,22 +69,63 @@ func ProcessEmbedDirectives(sqlPath, sql string) (string, error) {
|
|||||||
return "", fmt.Errorf("sql path is required for embed directives")
|
return "", fmt.Errorf("sql path is required for embed directives")
|
||||||
}
|
}
|
||||||
|
|
||||||
result := embedDirectivePattern.ReplaceAllString(sql, "")
|
assetRoot := options.AssetRoot
|
||||||
for i, directive := range directives {
|
if assetRoot == "" {
|
||||||
literal, placeholder, err := embedDirectiveLiteral(sqlPath, directive[1], i+1)
|
assetRoot = filepath.Dir(sqlPath)
|
||||||
|
}
|
||||||
|
absRoot, err := filepath.Abs(assetRoot)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return "", err
|
return "", err
|
||||||
}
|
}
|
||||||
if !embedPlaceholderPattern(placeholder).MatchString(result) {
|
absSQL, err := filepath.Abs(sqlPath)
|
||||||
return "", fmt.Errorf("%s embed directive %d: placeholder %s not found", sqlPath, i+1, placeholder)
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
scriptRel, err := filepath.Rel(absRoot, absSQL)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
if _, err := safefile.Relative(filepath.ToSlash(scriptRel)); err != nil {
|
||||||
|
return "", fmt.Errorf("SQL source path is outside asset root: %w", err)
|
||||||
|
}
|
||||||
|
root, err := openAssetRoot(absRoot)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
defer root.Close()
|
||||||
|
limit := options.MaxFileBytes
|
||||||
|
if limit == 0 {
|
||||||
|
limit = DefaultMaxAssetBytes
|
||||||
|
}
|
||||||
|
literals := map[string]string{}
|
||||||
|
used := map[string]bool{}
|
||||||
|
for i, directive := range directives {
|
||||||
|
literal, placeholder, err := embedDirectiveLiteral(root, filepath.Dir(scriptRel), limit, sqlPath, directive, i+1)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
if _, exists := literals[placeholder]; exists {
|
||||||
|
return "", fmt.Errorf("duplicate embed placeholder %s", placeholder)
|
||||||
|
}
|
||||||
|
literals[placeholder] = literal
|
||||||
|
}
|
||||||
|
result = sqllex.Walk(result, func(name string) string {
|
||||||
|
if literal, ok := literals[name]; ok {
|
||||||
|
used[name] = true
|
||||||
|
return literal
|
||||||
|
}
|
||||||
|
return name
|
||||||
|
}, nil)
|
||||||
|
for name := range literals {
|
||||||
|
if !used[name] {
|
||||||
|
return "", fmt.Errorf("%s: embed placeholder %s not found in executable SQL", sqlPath, name)
|
||||||
}
|
}
|
||||||
result = replaceEmbedPlaceholder(result, placeholder, literal)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
return result, nil
|
return result, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func embedDirectiveLiteral(sqlPath, raw string, directiveNumber int) (literal, placeholder string, err error) {
|
func embedDirectiveLiteral(root *os.Root, scriptDir string, maxBytes int64, sqlPath, raw string, directiveNumber int) (literal, placeholder string, err error) {
|
||||||
attrs, err := parseEmbedAttrs(raw)
|
attrs, err := parseEmbedAttrs(raw)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return "", "", fmt.Errorf("%s embed directive %d: %w", sqlPath, directiveNumber, err)
|
return "", "", fmt.Errorf("%s embed directive %d: %w", sqlPath, directiveNumber, err)
|
||||||
@@ -70,8 +144,12 @@ func embedDirectiveLiteral(sqlPath, raw string, directiveNumber int) (literal, p
|
|||||||
return "", "", fmt.Errorf("%s embed directive %d: mode must be text or base64", sqlPath, directiveNumber)
|
return "", "", fmt.Errorf("%s embed directive %d: mode must be text or base64", sqlPath, directiveNumber)
|
||||||
}
|
}
|
||||||
|
|
||||||
resolved := filepath.Join(filepath.Dir(sqlPath), filepath.Clean(pathValue))
|
relative, err := safefile.Relative(pathValue)
|
||||||
data, err := os.ReadFile(resolved)
|
if err != nil {
|
||||||
|
return "", "", fmt.Errorf("%s embed directive %d: %w", sqlPath, directiveNumber, err)
|
||||||
|
}
|
||||||
|
resolved := filepath.Join(scriptDir, relative)
|
||||||
|
data, err := safefile.Read(root, resolved, maxBytes)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return "", "", fmt.Errorf("%s embed directive %d: reading %s: %w", sqlPath, directiveNumber, resolved, err)
|
return "", "", fmt.Errorf("%s embed directive %d: reading %s: %w", sqlPath, directiveNumber, resolved, err)
|
||||||
}
|
}
|
||||||
@@ -153,13 +231,10 @@ func sqlStringLiteral(value string) (string, error) {
|
|||||||
if strings.ContainsRune(value, '\x00') {
|
if strings.ContainsRune(value, '\x00') {
|
||||||
return "", fmt.Errorf("embedded text contains NUL byte")
|
return "", fmt.Errorf("embedded text contains NUL byte")
|
||||||
}
|
}
|
||||||
return "'" + strings.ReplaceAll(value, "'", "''") + "'", nil
|
prefix := ""
|
||||||
|
if strings.Contains(value, "\\") {
|
||||||
|
prefix = "E"
|
||||||
|
value = strings.ReplaceAll(value, "\\", "\\\\")
|
||||||
}
|
}
|
||||||
|
return prefix + "'" + strings.ReplaceAll(value, "'", "''") + "'", nil
|
||||||
func replaceEmbedPlaceholder(sql, placeholder, literal string) string {
|
|
||||||
return embedPlaceholderPattern(placeholder).ReplaceAllString(sql, "${1}"+literal+"${2}")
|
|
||||||
}
|
|
||||||
|
|
||||||
func embedPlaceholderPattern(placeholder string) *regexp.Regexp {
|
|
||||||
return regexp.MustCompile(`(^|[^a-zA-Z0-9_:])` + regexp.QuoteMeta(placeholder) + `([^a-zA-Z0-9_]|$)`)
|
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -141,3 +141,114 @@ SELECT :body, :body_extra;
|
|||||||
t.Fatalf("placeholder boundary was not respected:\n%s", got)
|
t.Fatalf("placeholder boundary was not respected:\n%s", got)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestEmbedRejectsTraversalAbsolutePathsAndSymlinks(t *testing.T) {
|
||||||
|
root := t.TempDir()
|
||||||
|
scriptDir := filepath.Join(root, "scripts")
|
||||||
|
if err := os.Mkdir(scriptDir, 0o755); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
secret := filepath.Join(root, "secret.txt")
|
||||||
|
if err := os.WriteFile(secret, []byte("never print this secret"), 0o600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
for _, path := range []string{"../secret.txt", secret, `..\secret.txt`, `C:\secret.txt`, `\\server\secret.txt`} {
|
||||||
|
sql := "-- @embed: path='" + path + "' var=:body mode=text\nSELECT :body;"
|
||||||
|
got, err := ProcessEmbedDirectives(filepath.Join(scriptDir, "seed.sql"), sql)
|
||||||
|
if err == nil || got != "" {
|
||||||
|
t.Fatalf("unsafe embed path accepted %q: %q %v", path, got, err)
|
||||||
|
}
|
||||||
|
if strings.Contains(err.Error(), "never print this secret") {
|
||||||
|
t.Fatal("diagnostic leaked asset bytes")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if err := os.Symlink(root, filepath.Join(scriptDir, "link")); err != nil {
|
||||||
|
t.Skip(err)
|
||||||
|
}
|
||||||
|
if _, err := ProcessEmbedDirectives(filepath.Join(scriptDir, "seed.sql"), "-- @embed: path=link/secret.txt var=:body mode=text\nSELECT :body;"); err == nil {
|
||||||
|
t.Fatal("symlink parent must be rejected")
|
||||||
|
}
|
||||||
|
if err := os.Symlink(secret, filepath.Join(scriptDir, "leaf.txt")); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, err := ProcessEmbedDirectives(filepath.Join(scriptDir, "seed.sql"), "-- @embed: path=leaf.txt var=:body mode=text\nSELECT :body;"); err == nil {
|
||||||
|
t.Fatal("symlink asset must be rejected")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestEmbedExplicitRootAndSizeLimit(t *testing.T) {
|
||||||
|
root := t.TempDir()
|
||||||
|
dir := filepath.Join(root, "scripts")
|
||||||
|
if err := os.Mkdir(dir, 0o755); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(filepath.Join(dir, "body.txt"), []byte("12345"), 0o600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
sql := "-- @embed: path=body.txt var=:body mode=text\nSELECT :body;"
|
||||||
|
path := filepath.Join(dir, "seed.sql")
|
||||||
|
if _, err := ProcessEmbedDirectivesWithOptions(path, sql, EmbedOptions{AssetRoot: root, MaxFileBytes: 4}); err == nil || !strings.Contains(err.Error(), "byte limit") {
|
||||||
|
t.Fatalf("expected size limit error: %v", err)
|
||||||
|
}
|
||||||
|
if got, err := ProcessEmbedDirectivesWithOptions(path, sql, EmbedOptions{AssetRoot: root, MaxFileBytes: 5}); err != nil || !strings.Contains(got, "SELECT '12345';") {
|
||||||
|
t.Fatalf("valid bounded embed: %q %v", got, err)
|
||||||
|
}
|
||||||
|
if _, err := ProcessEmbedDirectivesWithOptions(path, sql, EmbedOptions{AssetRoot: t.TempDir()}); err == nil {
|
||||||
|
t.Fatal("source path outside explicit root must fail")
|
||||||
|
}
|
||||||
|
if _, err := ProcessEmbedDirectivesWithOptions(path, sql, EmbedOptions{MaxFileBytes: -1}); err == nil {
|
||||||
|
t.Fatal("negative limits must fail")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestManifestReadRechecksContainmentAndSize(t *testing.T) {
|
||||||
|
root, external := t.TempDir(), t.TempDir()
|
||||||
|
file := filepath.Join(root, "body.txt")
|
||||||
|
if err := os.WriteFile(file, []byte("valid"), 0o600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
item := Item{AssetRoot: root, MaxFileBytes: 4, Entry: ManifestEntry{File: file}}
|
||||||
|
if _, err := readItemAsset(item); err == nil {
|
||||||
|
t.Fatal("manifest size bound must apply at execution")
|
||||||
|
}
|
||||||
|
item.MaxFileBytes = 5
|
||||||
|
if got, err := readItemAsset(item); err != nil || string(got) != "valid" {
|
||||||
|
t.Fatalf("valid manifest asset: %q %v", got, err)
|
||||||
|
}
|
||||||
|
secret := filepath.Join(external, "secret.txt")
|
||||||
|
if err := os.WriteFile(secret, []byte("secret"), 0o600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := os.Remove(file); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := os.Symlink(secret, file); err != nil {
|
||||||
|
t.Skip(err)
|
||||||
|
}
|
||||||
|
if _, err := readItemAsset(item); err == nil {
|
||||||
|
t.Fatal("replacement symlink after scanning must fail")
|
||||||
|
}
|
||||||
|
item.Entry.File = secret
|
||||||
|
if _, err := readItemAsset(item); err == nil {
|
||||||
|
t.Fatal("forged asset path must fail")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestEmbedDefaultLimitRejectsOversizedFile(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
file, err := os.Create(filepath.Join(dir, "large.bin"))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := file.Truncate(DefaultMaxAssetBytes + 1); err != nil {
|
||||||
|
file.Close()
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := file.Close(); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
_, err = ProcessEmbedDirectives(filepath.Join(dir, "seed.sql"), "-- @embed: path=large.bin var=:payload mode=base64\nSELECT :payload;")
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "byte limit") {
|
||||||
|
t.Fatalf("expected default size bound: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -3,20 +3,13 @@ package assetloader
|
|||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"fmt"
|
"fmt"
|
||||||
"os"
|
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"regexp"
|
|
||||||
"strings"
|
"git.warky.dev/wdevs/relspecgo/pkg/sqllex"
|
||||||
|
|
||||||
"github.com/jackc/pgx/v5"
|
"github.com/jackc/pgx/v5"
|
||||||
)
|
)
|
||||||
|
|
||||||
// namedPlaceholder matches :identifier patterns (but not ::cast syntax).
|
|
||||||
var namedPlaceholder = regexp.MustCompile(`:([a-zA-Z_][a-zA-Z0-9_]*)`)
|
|
||||||
|
|
||||||
// pgCastMarker temporarily replaces :: to protect PostgreSQL cast syntax.
|
|
||||||
const pgCastMarker = "\x00PGCAST\x00"
|
|
||||||
|
|
||||||
// BuildQuery converts a SQL call that uses :name named placeholders into a
|
// BuildQuery converts a SQL call that uses :name named placeholders into a
|
||||||
// pgx-compatible positional-parameter query ($1, $2, …) and returns the
|
// pgx-compatible positional-parameter query ($1, $2, …) and returns the
|
||||||
// corresponding argument slice.
|
// corresponding argument slice.
|
||||||
@@ -30,13 +23,10 @@ const pgCastMarker = "\x00PGCAST\x00"
|
|||||||
// placeholder (not built-in and not in staticParams) returns an error.
|
// placeholder (not built-in and not in staticParams) returns an error.
|
||||||
// PostgreSQL cast syntax (::type) is left untouched.
|
// PostgreSQL cast syntax (::type) is left untouched.
|
||||||
func BuildQuery(call string, fileBytes []byte, filename string, staticParams map[string]string) (query string, args []any, err error) {
|
func BuildQuery(call string, fileBytes []byte, filename string, staticParams map[string]string) (query string, args []any, err error) {
|
||||||
// Protect :: casts before running the placeholder regex.
|
|
||||||
protected := strings.ReplaceAll(call, "::", pgCastMarker)
|
|
||||||
|
|
||||||
paramIndex := map[string]int{} // name → 1-based position
|
paramIndex := map[string]int{} // name → 1-based position
|
||||||
var firstErr error
|
var firstErr error
|
||||||
|
|
||||||
query = namedPlaceholder.ReplaceAllStringFunc(protected, func(match string) string {
|
query = sqllex.Walk(call, func(match string) string {
|
||||||
if firstErr != nil {
|
if firstErr != nil {
|
||||||
return match
|
return match
|
||||||
}
|
}
|
||||||
@@ -70,15 +60,12 @@ func BuildQuery(call string, fileBytes []byte, filename string, staticParams map
|
|||||||
paramIndex[name] = idx
|
paramIndex[name] = idx
|
||||||
args = append(args, val)
|
args = append(args, val)
|
||||||
return fmt.Sprintf("$%d", idx)
|
return fmt.Sprintf("$%d", idx)
|
||||||
})
|
}, nil)
|
||||||
|
|
||||||
if firstErr != nil {
|
if firstErr != nil {
|
||||||
return "", nil, firstErr
|
return "", nil, firstErr
|
||||||
}
|
}
|
||||||
|
|
||||||
// Restore :: casts.
|
|
||||||
query = strings.ReplaceAll(query, pgCastMarker, "::")
|
|
||||||
|
|
||||||
return query, args, nil
|
return query, args, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -86,7 +73,7 @@ func BuildQuery(call string, fileBytes []byte, filename string, staticParams map
|
|||||||
// absolute path set by ScanDir) and executes the configured SQL call via conn.
|
// absolute path set by ScanDir) and executes the configured SQL call via conn.
|
||||||
// The file's raw bytes are bound as a []byte parameter — no encoding or escaping.
|
// The file's raw bytes are bound as a []byte parameter — no encoding or escaping.
|
||||||
func ExecuteItem(ctx context.Context, conn *pgx.Conn, item Item) error {
|
func ExecuteItem(ctx context.Context, conn *pgx.Conn, item Item) error {
|
||||||
data, err := os.ReadFile(item.Entry.File)
|
data, err := readItemAsset(item)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("reading asset file %s: %w", item.Entry.File, err)
|
return fmt.Errorf("reading asset file %s: %w", item.Entry.File, err)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,36 @@
|
|||||||
|
package assetloader
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
|
||||||
|
"git.warky.dev/wdevs/relspecgo/pkg/safefile"
|
||||||
|
)
|
||||||
|
|
||||||
|
func openAssetRoot(path string) (*os.Root, error) {
|
||||||
|
if err := safefile.CheckRoot(path); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return os.OpenRoot(path)
|
||||||
|
}
|
||||||
|
|
||||||
|
func readItemAsset(item Item) ([]byte, error) {
|
||||||
|
if item.AssetRoot == "" {
|
||||||
|
return nil, fmt.Errorf("asset root is required for manifest execution")
|
||||||
|
}
|
||||||
|
root, err := openAssetRoot(item.AssetRoot)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
defer root.Close()
|
||||||
|
relative, err := filepath.Rel(item.AssetRoot, item.Entry.File)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
limit := item.MaxFileBytes
|
||||||
|
if limit == 0 {
|
||||||
|
limit = DefaultMaxAssetBytes
|
||||||
|
}
|
||||||
|
return safefile.Read(root, relative, limit)
|
||||||
|
}
|
||||||
+29
-21
@@ -18,9 +18,10 @@ import (
|
|||||||
"regexp"
|
"regexp"
|
||||||
"sort"
|
"sort"
|
||||||
"strconv"
|
"strconv"
|
||||||
"strings"
|
|
||||||
|
|
||||||
"gopkg.in/yaml.v3"
|
"gopkg.in/yaml.v3"
|
||||||
|
|
||||||
|
"git.warky.dev/wdevs/relspecgo/pkg/safefile"
|
||||||
)
|
)
|
||||||
|
|
||||||
// ManifestEntry describes a single file to load from an assets.yaml manifest.
|
// ManifestEntry describes a single file to load from an assets.yaml manifest.
|
||||||
@@ -37,6 +38,9 @@ type ManifestEntry struct {
|
|||||||
// Item combines a manifest entry with its ordering metadata and the resolved
|
// Item combines a manifest entry with its ordering metadata and the resolved
|
||||||
// directory where the manifest and asset file reside.
|
// directory where the manifest and asset file reside.
|
||||||
type Item struct {
|
type Item struct {
|
||||||
|
// AssetRoot is the trusted scan root, retained for validation at execution time.
|
||||||
|
AssetRoot string
|
||||||
|
MaxFileBytes int64
|
||||||
// Priority and Sequence come from the parent directory's naming pattern.
|
// Priority and Sequence come from the parent directory's naming pattern.
|
||||||
Priority int
|
Priority int
|
||||||
Sequence uint
|
Sequence uint
|
||||||
@@ -57,7 +61,12 @@ var dirPattern = regexp.MustCompile(`^(\d+)[_-](\d+)[_-](.+)$`)
|
|||||||
// absent or contains invalid YAML.
|
// absent or contains invalid YAML.
|
||||||
func LoadManifest(dir string) ([]ManifestEntry, error) {
|
func LoadManifest(dir string) ([]ManifestEntry, error) {
|
||||||
manifestPath := filepath.Join(dir, "assets.yaml")
|
manifestPath := filepath.Join(dir, "assets.yaml")
|
||||||
data, err := os.ReadFile(manifestPath)
|
root, err := openAssetRoot(dir)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
defer root.Close()
|
||||||
|
data, err := safefile.Read(root, "assets.yaml", DefaultMaxAssetBytes)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("reading %s: %w", manifestPath, err)
|
return nil, fmt.Errorf("reading %s: %w", manifestPath, err)
|
||||||
}
|
}
|
||||||
@@ -92,7 +101,7 @@ func ScanDir(baseDir string) ([]Item, error) {
|
|||||||
if d.IsDir() {
|
if d.IsDir() {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
if d.Name() != "assets.yaml" {
|
if d.Type()&os.ModeSymlink != 0 || d.Name() != "assets.yaml" {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -119,6 +128,8 @@ func ScanDir(baseDir string) ([]Item, error) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
items = append(items, Item{
|
items = append(items, Item{
|
||||||
|
AssetRoot: absBase,
|
||||||
|
MaxFileBytes: DefaultMaxAssetBytes,
|
||||||
Priority: priority,
|
Priority: priority,
|
||||||
Sequence: sequence,
|
Sequence: sequence,
|
||||||
DirName: dirName,
|
DirName: dirName,
|
||||||
@@ -170,31 +181,28 @@ func parseDirName(dir string) (priority int, sequence uint, name string) {
|
|||||||
// Returns the absolute path, a skip flag (true when the entry should be silently
|
// Returns the absolute path, a skip flag (true when the entry should be silently
|
||||||
// dropped), and any hard error.
|
// dropped), and any hard error.
|
||||||
func resolveAssetPath(absBase, manifestDir, file string) (absPath string, skip bool, err error) {
|
func resolveAssetPath(absBase, manifestDir, file string) (absPath string, skip bool, err error) {
|
||||||
// Clean and join before any symlink resolution so we can detect traversal.
|
if _, err := safefile.Relative(file); err != nil {
|
||||||
joined := filepath.Join(manifestDir, filepath.Clean(file))
|
return "", true, nil
|
||||||
|
}
|
||||||
// Ensure the cleaned path is still inside absBase.
|
joined := filepath.Join(manifestDir, file)
|
||||||
rel, err := filepath.Rel(absBase, joined)
|
rel, err := filepath.Rel(absBase, joined)
|
||||||
if err != nil || strings.HasPrefix(rel, "..") {
|
if err != nil {
|
||||||
// Path escapes the base directory; skip silently.
|
|
||||||
return "", true, nil
|
return "", true, nil
|
||||||
}
|
}
|
||||||
|
root, err := openAssetRoot(absBase)
|
||||||
// Walk each component to detect symlinks.
|
if err != nil {
|
||||||
parts := strings.Split(rel, string(filepath.Separator))
|
return "", false, err
|
||||||
current := absBase
|
}
|
||||||
for _, part := range parts {
|
defer root.Close()
|
||||||
current = filepath.Join(current, part)
|
if err := safefile.Check(root, rel); err != nil {
|
||||||
info, statErr := os.Lstat(current)
|
|
||||||
if statErr != nil {
|
|
||||||
// File doesn't exist; skip.
|
|
||||||
return "", true, nil
|
return "", true, nil
|
||||||
}
|
}
|
||||||
if info.Mode()&os.ModeSymlink != 0 {
|
info, err := root.Stat(rel)
|
||||||
// Symlink in path; skip silently.
|
if err != nil || !info.Mode().IsRegular() {
|
||||||
return "", true, nil
|
return "", true, nil
|
||||||
}
|
}
|
||||||
|
if info.Size() > DefaultMaxAssetBytes {
|
||||||
|
return "", false, fmt.Errorf("asset %q exceeds %d byte limit", rel, DefaultMaxAssetBytes)
|
||||||
}
|
}
|
||||||
|
|
||||||
return joined, false, nil
|
return joined, false, nil
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,60 @@
|
|||||||
|
package assetloader
|
||||||
|
|
||||||
|
import (
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestBuildQuerySQLContexts(t *testing.T) {
|
||||||
|
call := `SELECT ':unknown', ":unknown", $$:unknown$$, E'it\':unknown', :bytes::bytea, :filename, :bytes /* :unknown /* :unknown */ */ -- :unknown
|
||||||
|
`
|
||||||
|
got, args, err := BuildQuery(call, []byte("$1"), "File.txt", nil)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(args) != 2 || !strings.Contains(got, "$1::bytea, $2, $1") || !strings.Contains(got, "$$:unknown$$") {
|
||||||
|
t.Fatalf("%s %#v", got, args)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestEmbedLiteralOnePassAndSQLContexts(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
data := "price $1 ${2} $HOME $$ :b 'quote' \\path"
|
||||||
|
for file, text := range map[string]string{"a.txt": data, "b.txt": "Second"} {
|
||||||
|
if err := os.WriteFile(filepath.Join(dir, file), []byte(text), 0o600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
sql := `-- @embed: path=a.txt var=:a mode=text
|
||||||
|
-- @embed: path=b.txt var=:b mode=text
|
||||||
|
SELECT :a,:a,:b, ':a', ":a", $$:a$$, 1::integer; -- :a
|
||||||
|
/* :a */`
|
||||||
|
got, err := ProcessEmbedDirectives(filepath.Join(dir, "script.sql"), sql)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
literal, err := sqlStringLiteral(data)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if strings.Count(got, literal) != 2 || !strings.Contains(got, "'Second'") || !strings.Contains(got, "':a'") || !strings.Contains(got, "$$:a$$") {
|
||||||
|
t.Fatalf("%s", got)
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, sql := range []string{"SELECT $body$\n-- @embed: path=missing var=:x mode=text\n$body$", "SELECT '\n-- @embed: path=missing var=:x mode=text\n'", "/*\n-- @embed: path=missing var=:x mode=text\n*/", "SELECT 1; -- @embed: path=missing var=:x mode=text"} {
|
||||||
|
if HasEmbedDirectives(sql) {
|
||||||
|
t.Fatalf("false directive: %s", sql)
|
||||||
|
}
|
||||||
|
got, err := ProcessEmbedDirectives("", sql)
|
||||||
|
if err != nil || got != sql {
|
||||||
|
t.Fatalf("%q: %v", got, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, sql := range []string{"-- @embed: path=a.txt var=:a mode=text\nSELECT ':a'", "-- @embed: path=a.txt var=:a mode=text\n-- @embed: path=b.txt var=:a mode=text\nSELECT :a"} {
|
||||||
|
if _, err := ProcessEmbedDirectives(filepath.Join(dir, "script.sql"), sql); err == nil {
|
||||||
|
t.Fatal("expected invalid directive use")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -277,6 +277,8 @@ func parseHumanSize(s string) (int64, error) {
|
|||||||
|
|
||||||
// Options carries the subset of command flags a job file may set.
|
// Options carries the subset of command flags a job file may set.
|
||||||
type Options struct {
|
type Options struct {
|
||||||
|
StrictConversion bool `yaml:"strict_conversion"`
|
||||||
|
LowercaseIdentifiers bool `yaml:"lowercase_identifiers"`
|
||||||
FlattenSchema bool `yaml:"flatten_schema"`
|
FlattenSchema bool `yaml:"flatten_schema"`
|
||||||
Schema string `yaml:"schema"`
|
Schema string `yaml:"schema"`
|
||||||
Package string `yaml:"package"`
|
Package string `yaml:"package"`
|
||||||
@@ -285,6 +287,7 @@ type Options struct {
|
|||||||
Types string `yaml:"types"`
|
Types string `yaml:"types"`
|
||||||
ArrayNullable string `yaml:"array_nullable"`
|
ArrayNullable string `yaml:"array_nullable"`
|
||||||
ContinueOnError bool `yaml:"continue_on_error"`
|
ContinueOnError bool `yaml:"continue_on_error"`
|
||||||
|
AtomicExecution bool `yaml:"atomic_execution"`
|
||||||
// FullDDL makes direct pgsql output execute the full idempotent DDL instead of
|
// FullDDL makes direct pgsql output execute the full idempotent DDL instead of
|
||||||
// diffing against the live database first.
|
// diffing against the live database first.
|
||||||
FullDDL bool `yaml:"full_ddl"`
|
FullDDL bool `yaml:"full_ddl"`
|
||||||
@@ -479,6 +482,14 @@ func (s *Set) Validate() error {
|
|||||||
|
|
||||||
func (j *Job) validate() []string {
|
func (j *Job) validate() []string {
|
||||||
var e []string
|
var e []string
|
||||||
|
if j.Options.AtomicExecution {
|
||||||
|
if j.Options.ContinueOnError {
|
||||||
|
e = append(e, "atomic_execution cannot be combined with continue_on_error")
|
||||||
|
}
|
||||||
|
if (j.Command != CommandConvert && j.Command != CommandMerge) || j.Output == nil || j.Output.ConnEnv == "" || j.Output.Format != "pgsql" {
|
||||||
|
e = append(e, "atomic_execution requires convert/merge with pgsql database output")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
switch j.Command {
|
switch j.Command {
|
||||||
case CommandConvert, CommandMerge, CommandScriptsList, CommandScriptsExec,
|
case CommandConvert, CommandMerge, CommandScriptsList, CommandScriptsExec,
|
||||||
|
|||||||
+19
-19
@@ -61,6 +61,7 @@ var MariaDBToCanonicalTypes = map[string]string{
|
|||||||
// CanonicalToMariaDBTypes maps canonical types to MariaDB/MySQL types.
|
// CanonicalToMariaDBTypes maps canonical types to MariaDB/MySQL types.
|
||||||
var CanonicalToMariaDBTypes = map[string]string{
|
var CanonicalToMariaDBTypes = map[string]string{
|
||||||
"bool": "TINYINT(1)",
|
"bool": "TINYINT(1)",
|
||||||
|
"boolean": "TINYINT(1)",
|
||||||
"int8": "TINYINT",
|
"int8": "TINYINT",
|
||||||
"int16": "SMALLINT",
|
"int16": "SMALLINT",
|
||||||
"int": "INT",
|
"int": "INT",
|
||||||
@@ -114,22 +115,28 @@ func NormalizeMariaDBType(baseType string) string {
|
|||||||
// ConvertMariaDBToCanonical converts a MariaDB/MySQL type name to the canonical type.
|
// ConvertMariaDBToCanonical converts a MariaDB/MySQL type name to the canonical type.
|
||||||
// Strips dimension parameters and normalizes aliases. Defaults to "string".
|
// Strips dimension parameters and normalizes aliases. Defaults to "string".
|
||||||
func ConvertMariaDBToCanonical(mariadbType string) string {
|
func ConvertMariaDBToCanonical(mariadbType string) string {
|
||||||
base := strings.ToLower(strings.TrimSpace(mariadbType))
|
lower := strings.ToLower(strings.TrimSpace(mariadbType))
|
||||||
if idx := strings.Index(base, "("); idx >= 0 {
|
base := lower
|
||||||
base = strings.TrimSpace(base[:idx])
|
if idx := strings.IndexAny(base, "( "); idx >= 0 {
|
||||||
|
base = base[:idx]
|
||||||
|
}
|
||||||
|
canonical, ok := MariaDBToCanonicalTypes[base]
|
||||||
|
if ok {
|
||||||
|
if strings.Contains(lower, "unsigned") || strings.Contains(lower, "zerofill") {
|
||||||
|
switch canonical {
|
||||||
|
case "int8":
|
||||||
|
return "uint8"
|
||||||
|
case "int16":
|
||||||
|
return "uint16"
|
||||||
|
case "int":
|
||||||
|
return "uint"
|
||||||
|
case "int64":
|
||||||
|
return "uint64"
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if canonical, ok := MariaDBToCanonicalTypes[base]; ok {
|
|
||||||
return canonical
|
return canonical
|
||||||
}
|
}
|
||||||
|
|
||||||
// Prefix match for composite types (e.g., "unsigned bigint")
|
|
||||||
for key, canonical := range MariaDBToCanonicalTypes {
|
|
||||||
if strings.HasPrefix(base, key) {
|
|
||||||
return canonical
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return "string"
|
return "string"
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -145,12 +152,5 @@ func ConvertCanonicalToMariaDB(canonicalType string) string {
|
|||||||
return mariadbType
|
return mariadbType
|
||||||
}
|
}
|
||||||
|
|
||||||
// Prefix fallback
|
|
||||||
for canonical, mariadb := range CanonicalToMariaDBTypes {
|
|
||||||
if strings.HasPrefix(lower, canonical) {
|
|
||||||
return mariadb
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return "VARCHAR(255)"
|
return "VARCHAR(255)"
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,13 @@
|
|||||||
|
package mariadb
|
||||||
|
|
||||||
|
import "testing"
|
||||||
|
|
||||||
|
func TestIntegerSignedness(t *testing.T) {
|
||||||
|
for _, tc := range []struct{ input, want string }{
|
||||||
|
{"tinyint(3) unsigned", "uint8"}, {"smallint unsigned", "uint16"}, {"mediumint unsigned", "uint"}, {"int unsigned", "uint"}, {"bigint unsigned", "uint64"}, {"bigint zerofill", "uint64"}, {"bigint", "int64"}, {"varchar(32)", "string"},
|
||||||
|
} {
|
||||||
|
if got := ConvertMariaDBToCanonical(tc.input); got != tc.want {
|
||||||
|
t.Errorf("%s: got %s want %s", tc.input, got, tc.want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -256,9 +256,19 @@ func (d *Column) SQLName() string {
|
|||||||
return strings.ToLower(d.Name)
|
return strings.ToLower(d.Name)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// IndexPart is an ordered MySQL index key part. Name and Expression are mutually
|
||||||
|
// exclusive; expressions are executable SQL and are not identifier-normalized.
|
||||||
|
type IndexPart struct {
|
||||||
|
Name string `json:"name,omitempty" yaml:"name,omitempty" xml:"name,omitempty"`
|
||||||
|
Expression string `json:"expression,omitempty" yaml:"expression,omitempty" xml:"expression,omitempty"`
|
||||||
|
Length int `json:"length,omitempty" yaml:"length,omitempty" xml:"length,omitempty"`
|
||||||
|
Descending bool `json:"descending,omitempty" yaml:"descending,omitempty" xml:"descending,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
// Index represents a database index for optimizing query performance.
|
// Index represents a database index for optimizing query performance.
|
||||||
// Indexes can be unique, partial, or include additional columns.
|
// Indexes can be unique, partial, or include additional columns.
|
||||||
type Index struct {
|
type Index struct {
|
||||||
|
KeyParts []IndexPart `json:"key_parts,omitempty" yaml:"key_parts,omitempty" xml:"key_parts,omitempty"`
|
||||||
Name string `json:"name" yaml:"name" xml:"name"`
|
Name string `json:"name" yaml:"name" xml:"name"`
|
||||||
Description string `json:"description,omitempty" yaml:"description,omitempty" xml:"description,omitempty"`
|
Description string `json:"description,omitempty" yaml:"description,omitempty" xml:"description,omitempty"`
|
||||||
Table string `json:"table,omitempty" yaml:"table,omitempty" xml:"table,omitempty"`
|
Table string `json:"table,omitempty" yaml:"table,omitempty" xml:"table,omitempty"`
|
||||||
@@ -318,6 +328,7 @@ func (d *Relationship) SQLName() string {
|
|||||||
// Constraint represents a database constraint that enforces data integrity rules.
|
// Constraint represents a database constraint that enforces data integrity rules.
|
||||||
// Constraints can be primary keys, foreign keys, unique constraints, check constraints, or not-null constraints.
|
// Constraints can be primary keys, foreign keys, unique constraints, check constraints, or not-null constraints.
|
||||||
type Constraint struct {
|
type Constraint struct {
|
||||||
|
Metadata map[string]any `json:"metadata,omitempty" yaml:"metadata,omitempty" xml:"-"`
|
||||||
Name string `json:"name" yaml:"name" xml:"name"`
|
Name string `json:"name" yaml:"name" xml:"name"`
|
||||||
Type ConstraintType `json:"type" yaml:"type" xml:"type"`
|
Type ConstraintType `json:"type" yaml:"type" xml:"type"`
|
||||||
Columns []string `json:"columns" yaml:"columns" xml:"columns"`
|
Columns []string `json:"columns" yaml:"columns" xml:"columns"`
|
||||||
|
|||||||
@@ -0,0 +1,194 @@
|
|||||||
|
package mysql
|
||||||
|
|
||||||
|
import (
|
||||||
|
"regexp"
|
||||||
|
"strings"
|
||||||
|
)
|
||||||
|
|
||||||
|
// columnDefaults extracts native DEFAULT expressions from SHOW CREATE TABLE.
|
||||||
|
// MySQL can truncate binary defaults in information_schema (for example X'00FF'
|
||||||
|
// appears as just "0x"). SHOW CREATE is authoritative for their SQL spelling.
|
||||||
|
// Separators and whitespace inside quoted strings, identifiers and nested
|
||||||
|
// expressions are never treated as definition/default boundaries.
|
||||||
|
func columnDefaults(create string) map[string]string {
|
||||||
|
out := map[string]string{}
|
||||||
|
depth, start := 0, 0
|
||||||
|
var quoted byte
|
||||||
|
definitions := []string{}
|
||||||
|
for i := 0; i < len(create); i++ {
|
||||||
|
ch := create[i]
|
||||||
|
if quoted != 0 {
|
||||||
|
if ch == '\\' && quoted != '`' && i+1 < len(create) {
|
||||||
|
i++
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if ch == quoted {
|
||||||
|
if i+1 < len(create) && create[i+1] == quoted {
|
||||||
|
i++
|
||||||
|
} else {
|
||||||
|
quoted = 0
|
||||||
|
}
|
||||||
|
}
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if ch == '\'' || ch == '"' || ch == '`' {
|
||||||
|
quoted = ch
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
switch ch {
|
||||||
|
case '(':
|
||||||
|
depth++
|
||||||
|
if depth == 1 {
|
||||||
|
start = i + 1
|
||||||
|
}
|
||||||
|
case ',':
|
||||||
|
if depth == 1 {
|
||||||
|
definitions = append(definitions, create[start:i])
|
||||||
|
start = i + 1
|
||||||
|
}
|
||||||
|
case ')':
|
||||||
|
depth--
|
||||||
|
if depth == 0 {
|
||||||
|
definitions = append(definitions, create[start:i])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if ch == ')' && depth == 0 {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, def := range definitions {
|
||||||
|
def = strings.TrimSpace(def)
|
||||||
|
if len(def) == 0 || def[0] != '`' {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
var name strings.Builder
|
||||||
|
end := 1
|
||||||
|
for end < len(def) {
|
||||||
|
if def[end] == '`' {
|
||||||
|
if end+1 < len(def) && def[end+1] == '`' {
|
||||||
|
name.WriteByte('`')
|
||||||
|
end += 2
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
end++
|
||||||
|
break
|
||||||
|
}
|
||||||
|
name.WriteByte(def[end])
|
||||||
|
end++
|
||||||
|
}
|
||||||
|
// Consume full tokens so DEFAULT inside a generated expression or literal
|
||||||
|
// cannot masquerade as a column default.
|
||||||
|
for end < len(def) {
|
||||||
|
for end < len(def) && isSpace(def[end]) {
|
||||||
|
end++
|
||||||
|
}
|
||||||
|
begin := end
|
||||||
|
end = tokenEnd(def, end)
|
||||||
|
if end == begin {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
if strings.EqualFold(def[begin:end], "DEFAULT") {
|
||||||
|
for end < len(def) && isSpace(def[end]) {
|
||||||
|
end++
|
||||||
|
}
|
||||||
|
finish := tokenEnd(def, end)
|
||||||
|
out[name.String()] = def[end:finish]
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
func isSpace(b byte) bool { return b == ' ' || b == '\n' || b == '\r' || b == '\t' }
|
||||||
|
func tokenEnd(s string, start int) int {
|
||||||
|
depth := 0
|
||||||
|
var quoted byte
|
||||||
|
for i := start; i < len(s); i++ {
|
||||||
|
ch := s[i]
|
||||||
|
if quoted != 0 {
|
||||||
|
if ch == '\\' && quoted != '`' && i+1 < len(s) {
|
||||||
|
i++
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if ch == quoted {
|
||||||
|
if i+1 < len(s) && s[i+1] == quoted {
|
||||||
|
i++
|
||||||
|
} else {
|
||||||
|
quoted = 0
|
||||||
|
}
|
||||||
|
}
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if ch == '\'' || ch == '"' || ch == '`' {
|
||||||
|
quoted = ch
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if ch == '(' {
|
||||||
|
depth++
|
||||||
|
}
|
||||||
|
if ch == ')' {
|
||||||
|
depth--
|
||||||
|
}
|
||||||
|
if depth == 0 && isSpace(ch) {
|
||||||
|
return i
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return len(s)
|
||||||
|
}
|
||||||
|
|
||||||
|
// SHOW CREATE has native options not exposed by STATISTICS/COLUMNS. Ignore
|
||||||
|
// quoted identifiers and values when detecting options outside our contract.
|
||||||
|
func unquotedSQL(create string) string {
|
||||||
|
var code strings.Builder
|
||||||
|
var quoted byte
|
||||||
|
for i := 0; i < len(create); i++ {
|
||||||
|
ch := create[i]
|
||||||
|
if quoted != 0 {
|
||||||
|
if ch == '\\' && quoted != '`' && i+1 < len(create) {
|
||||||
|
i++
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if ch == quoted {
|
||||||
|
if i+1 < len(create) && create[i+1] == quoted {
|
||||||
|
i++
|
||||||
|
} else {
|
||||||
|
quoted = 0
|
||||||
|
code.WriteByte(' ')
|
||||||
|
}
|
||||||
|
}
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if ch == '\'' || ch == '"' || ch == '`' {
|
||||||
|
quoted = ch
|
||||||
|
code.WriteByte(' ')
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
code.WriteByte(ch)
|
||||||
|
}
|
||||||
|
return code.String()
|
||||||
|
}
|
||||||
|
|
||||||
|
var autoIncrementOption = regexp.MustCompile(`(?i)\bAUTO_INCREMENT\s*=\s*([0-9]+)`)
|
||||||
|
|
||||||
|
func autoIncrementValue(create string) string {
|
||||||
|
if match := autoIncrementOption.FindStringSubmatch(unquotedSQL(create)); len(match) > 1 {
|
||||||
|
return match[1]
|
||||||
|
}
|
||||||
|
return "1"
|
||||||
|
}
|
||||||
|
|
||||||
|
func unsupportedNativeOptions(create string) []string {
|
||||||
|
words := strings.FieldsFunc(strings.ToUpper(unquotedSQL(create)), func(r rune) bool { return (r < 'A' || r > 'Z') && r != '_' })
|
||||||
|
var out []string
|
||||||
|
for i, w := range words {
|
||||||
|
switch w {
|
||||||
|
case "ENGINE_ATTRIBUTE", "SECONDARY_ENGINE_ATTRIBUTE", "KEY_BLOCK_SIZE", "COLUMN_FORMAT", "STORAGE":
|
||||||
|
out = append(out, w)
|
||||||
|
case "PARSER":
|
||||||
|
if i > 0 && words[i-1] == "WITH" {
|
||||||
|
out = append(out, "WITH PARSER")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
@@ -0,0 +1,54 @@
|
|||||||
|
package mysql
|
||||||
|
|
||||||
|
import "testing"
|
||||||
|
|
||||||
|
func TestUnescapeExpression(t *testing.T) {
|
||||||
|
for _, tc := range []struct{ input, want string }{
|
||||||
|
{`_utf8mb4\'one\\ntwo\'`, `_utf8mb4'one\ntwo'`},
|
||||||
|
{`concat(\'don\\\'t\',\' change\')`, `concat('don\'t',' change')`},
|
||||||
|
{`(uuid())`, `(uuid())`},
|
||||||
|
} {
|
||||||
|
if got := unescapeExpression(tc.input); got != tc.want {
|
||||||
|
t.Errorf("%q: got %q want %q", tc.input, got, tc.want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestNativeColumnDefaults(t *testing.T) {
|
||||||
|
create := "CREATE TABLE `odd(table` (\n" +
|
||||||
|
" `a,b` varbinary(8) DEFAULT 0x0027FF,\n" +
|
||||||
|
" `tick``name` varchar(30) DEFAULT 'don''t, (split)',\n" +
|
||||||
|
" `expr` varchar(30) DEFAULT (concat('DEFAULT, ', (uuid()))),\n" +
|
||||||
|
" `generated` varchar(30) GENERATED ALWAYS AS (concat('DEFAULT',`expr`)) VIRTUAL,\n" +
|
||||||
|
" `new\nline` int DEFAULT 3,\n" +
|
||||||
|
" PRIMARY KEY (`a,b`)\n) ENGINE=InnoDB"
|
||||||
|
got := columnDefaults(create)
|
||||||
|
want := map[string]string{"a,b": "0x0027FF", "tick`name": "'don''t, (split)'", "expr": "(concat('DEFAULT, ', (uuid())))", "new\nline": "3"}
|
||||||
|
if len(got) != len(want) {
|
||||||
|
t.Fatalf("defaults: %v", got)
|
||||||
|
}
|
||||||
|
for name, value := range want {
|
||||||
|
if got[name] != value {
|
||||||
|
t.Errorf("%s: got %q want %q", name, got[name], value)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestNativeOptionsIgnoreQuotedValues(t *testing.T) {
|
||||||
|
create := "CREATE TABLE `WITH PARSER` (`STORAGE` int COMMENT 'KEY_BLOCK_SIZE', KEY `ENGINE_ATTRIBUTE` (`STORAGE`) WITH PARSER `ngram`) ENGINE=InnoDB"
|
||||||
|
options := unsupportedNativeOptions(create)
|
||||||
|
if len(options) != 1 || options[0] != "WITH PARSER" {
|
||||||
|
t.Fatalf("native options: %v", options)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestNativeAutoIncrementIgnoresQuotedValues(t *testing.T) {
|
||||||
|
for _, tc := range []struct{ create, want string }{
|
||||||
|
{"CREATE TABLE `x` (`id` bigint AUTO_INCREMENT) ENGINE=InnoDB AUTO_INCREMENT=18446744073709551615", "18446744073709551615"},
|
||||||
|
{"CREATE TABLE `x` (`id` bigint AUTO_INCREMENT) COMMENT='AUTO_INCREMENT=12'", "1"},
|
||||||
|
} {
|
||||||
|
if got := autoIncrementValue(tc.create); got != tc.want {
|
||||||
|
t.Errorf("got %s want %s", got, tc.want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
+252
-43
@@ -4,9 +4,10 @@ import (
|
|||||||
"context"
|
"context"
|
||||||
"database/sql"
|
"database/sql"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"regexp"
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
_ "github.com/go-sql-driver/mysql"
|
mysqlDriver "github.com/go-sql-driver/mysql"
|
||||||
|
|
||||||
"git.warky.dev/wdevs/relspecgo/pkg/mariadb"
|
"git.warky.dev/wdevs/relspecgo/pkg/mariadb"
|
||||||
"git.warky.dev/wdevs/relspecgo/pkg/models"
|
"git.warky.dev/wdevs/relspecgo/pkg/models"
|
||||||
@@ -31,12 +32,13 @@ func (r *Reader) ReadDatabase() (*models.Database, error) {
|
|||||||
return nil, fmt.Errorf("failed to connect: %w", err)
|
return nil, fmt.Errorf("failed to connect: %w", err)
|
||||||
}
|
}
|
||||||
defer r.close()
|
defer r.close()
|
||||||
var name, version string
|
var name, version, sqlMode string
|
||||||
if err := r.db.QueryRowContext(r.ctx, "SELECT DATABASE()").Scan(&name); err != nil {
|
if err := r.db.QueryRowContext(r.ctx, "SELECT DATABASE(), @@SESSION.sql_mode").Scan(&name, &sqlMode); err != nil {
|
||||||
return nil, fmt.Errorf("failed to get database name: %w", err)
|
return nil, fmt.Errorf("failed to get database name: %w", err)
|
||||||
}
|
}
|
||||||
_ = r.db.QueryRowContext(r.ctx, "SELECT VERSION()").Scan(&version)
|
_ = r.db.QueryRowContext(r.ctx, "SELECT VERSION()").Scan(&version)
|
||||||
db := models.InitDatabase(name)
|
db := models.InitDatabase(name)
|
||||||
|
db.Metadata["mysql_sql_mode"] = sqlMode
|
||||||
db.DatabaseType, db.SourceFormat, db.DatabaseVersion = models.MySQLDatabaseType, "mysql", version
|
db.DatabaseType, db.SourceFormat, db.DatabaseVersion = models.MySQLDatabaseType, "mysql", version
|
||||||
schemas, err := r.querySchemas(name)
|
schemas, err := r.querySchemas(name)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -48,8 +50,10 @@ func (r *Reader) ReadDatabase() (*models.Database, error) {
|
|||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
schema.Tables = tables
|
schema.Tables = tables
|
||||||
|
schema.Metadata["mysql_sql_mode"] = sqlMode
|
||||||
for _, table := range tables {
|
for _, table := range tables {
|
||||||
table.Columns, err = r.queryColumns(schema.Name, table.Name)
|
table.Metadata["mysql_sql_mode"] = sqlMode
|
||||||
|
table.Columns, err = r.queryColumns(schema.Name, table)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
@@ -62,6 +66,15 @@ func (r *Reader) ReadDatabase() (*models.Database, error) {
|
|||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
table.RefSchema = schema
|
table.RefSchema = schema
|
||||||
|
for _, c := range table.Constraints {
|
||||||
|
if c.Type == models.PrimaryKeyConstraint {
|
||||||
|
for _, n := range c.Columns {
|
||||||
|
if col := table.Columns[n]; col != nil {
|
||||||
|
col.IsPrimaryKey = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
for _, c := range table.Constraints {
|
for _, c := range table.Constraints {
|
||||||
if c.Type == models.ForeignKeyConstraint {
|
if c.Type == models.ForeignKeyConstraint {
|
||||||
r.deriveRelationship(table, c)
|
r.deriveRelationship(table, c)
|
||||||
@@ -71,6 +84,9 @@ func (r *Reader) ReadDatabase() (*models.Database, error) {
|
|||||||
schema.RefDatabase = db
|
schema.RefDatabase = db
|
||||||
db.Schemas = append(db.Schemas, schema)
|
db.Schemas = append(db.Schemas, schema)
|
||||||
}
|
}
|
||||||
|
if err := r.readUnsupported(db, name); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
return db, nil
|
return db, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -97,7 +113,17 @@ func (r *Reader) ReadTable() (*models.Table, error) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (r *Reader) connect() error {
|
func (r *Reader) connect() error {
|
||||||
db, err := sql.Open("mysql", r.options.ConnectionString)
|
cfg, err := mysqlDriver.ParseDSN(r.options.ConnectionString)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if cfg.Params == nil {
|
||||||
|
cfg.Params = map[string]string{}
|
||||||
|
}
|
||||||
|
// SHOW CREATE parsing requires quoted column identifiers, including when the
|
||||||
|
// caller's metadata connection would otherwise disable show-create quoting.
|
||||||
|
cfg.Params["sql_quote_show_create"] = "1"
|
||||||
|
db, err := sql.Open("mysql", cfg.FormatDSN())
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
@@ -134,61 +160,147 @@ func (r *Reader) querySchemas(current string) ([]*models.Schema, error) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (r *Reader) queryTables(schema string) ([]*models.Table, error) {
|
func (r *Reader) queryTables(schema string) ([]*models.Table, error) {
|
||||||
rows, err := r.db.QueryContext(r.ctx, "SELECT TABLE_NAME FROM information_schema.TABLES WHERE TABLE_SCHEMA = ? AND TABLE_TYPE = 'BASE TABLE' ORDER BY TABLE_NAME", schema)
|
rows, err := r.db.QueryContext(r.ctx, "SELECT TABLE_NAME, ENGINE, TABLE_COLLATION, TABLE_COMMENT, CREATE_OPTIONS, AUTO_INCREMENT FROM information_schema.TABLES WHERE TABLE_SCHEMA = ? AND TABLE_TYPE = 'BASE TABLE' ORDER BY TABLE_NAME", schema)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
defer rows.Close()
|
defer rows.Close()
|
||||||
var out []*models.Table
|
var out []*models.Table
|
||||||
for rows.Next() {
|
for rows.Next() {
|
||||||
var n string
|
var n, engine, collation, comment, options string
|
||||||
if err := rows.Scan(&n); err != nil {
|
var auto sql.NullString
|
||||||
|
if err := rows.Scan(&n, &engine, &collation, &comment, &options, &auto); err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
out = append(out, models.InitTable(n, schema))
|
t := models.InitTable(n, schema)
|
||||||
|
t.Comment = comment
|
||||||
|
t.Metadata["mysql_engine"], t.Metadata["mysql_collation"], t.Metadata["mysql_create_options"] = engine, collation, options
|
||||||
|
if auto.Valid {
|
||||||
|
t.Metadata["mysql_auto_increment"] = auto.String
|
||||||
|
}
|
||||||
|
out = append(out, t)
|
||||||
}
|
}
|
||||||
return out, rows.Err()
|
return out, rows.Err()
|
||||||
}
|
}
|
||||||
|
|
||||||
func (r *Reader) queryColumns(schema, table string) (map[string]*models.Column, error) {
|
var onUpdateExtra = regexp.MustCompile(`on update current_timestamp(?:\(\d*\))?`)
|
||||||
rows, err := r.db.QueryContext(r.ctx, `SELECT COLUMN_NAME, COLUMN_TYPE, IS_NULLABLE, COLUMN_DEFAULT, ORDINAL_POSITION, EXTRA, COLUMN_COMMENT FROM information_schema.COLUMNS WHERE TABLE_SCHEMA = ? AND TABLE_NAME = ? ORDER BY ORDINAL_POSITION`, schema, table)
|
|
||||||
|
func (r *Reader) queryColumns(schema string, table *models.Table) (map[string]*models.Column, error) {
|
||||||
|
rows, err := r.db.QueryContext(r.ctx, `SELECT COLUMN_NAME, COLUMN_TYPE, IS_NULLABLE, COLUMN_DEFAULT, ORDINAL_POSITION, EXTRA, COLUMN_COMMENT,
|
||||||
|
CHARACTER_MAXIMUM_LENGTH, NUMERIC_PRECISION, NUMERIC_SCALE, DATETIME_PRECISION, COLLATION_NAME, CHARACTER_SET_NAME, GENERATION_EXPRESSION, SRS_ID
|
||||||
|
FROM information_schema.COLUMNS WHERE TABLE_SCHEMA=? AND TABLE_NAME=? ORDER BY ORDINAL_POSITION`, schema, table.Name)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, fmt.Errorf("columns %s.%s: %w", schema, table.Name, err)
|
||||||
}
|
}
|
||||||
defer rows.Close()
|
defer rows.Close()
|
||||||
out := map[string]*models.Column{}
|
out := map[string]*models.Column{}
|
||||||
for rows.Next() {
|
for rows.Next() {
|
||||||
var name, typ, nullable, extra, comment string
|
var name, typ, nullable, extra, comment string
|
||||||
var def sql.NullString
|
var def, collation, charset, generation sql.NullString
|
||||||
|
var length, precision, scale, datetime, srid sql.NullInt64
|
||||||
var pos int
|
var pos int
|
||||||
if err := rows.Scan(&name, &typ, &nullable, &def, &pos, &extra, &comment); err != nil {
|
if err := rows.Scan(&name, &typ, &nullable, &def, &pos, &extra, &comment, &length, &precision, &scale, &datetime, &collation, &charset, &generation, &srid); err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
c := models.InitColumn(name, table, schema)
|
c := models.InitColumn(name, table.Name, schema)
|
||||||
c.Type = r.mapDataType(typ)
|
c.Type, c.NotNull, c.Sequence, c.Comment = r.mapDataType(typ), nullable == "NO", uint(pos), comment
|
||||||
c.NotNull = strings.EqualFold(nullable, "NO")
|
// The native declaration preserves mediumint, enum/set, binary/blob sizes,
|
||||||
c.Sequence = uint(pos)
|
// unsigned/zerofill and timestamp versus datetime. Canonical dimensions remain editable.
|
||||||
c.Comment = comment
|
c.Length, c.Precision, c.Scale = int(length.Int64), int(precision.Int64), int(scale.Int64)
|
||||||
|
if datetime.Valid {
|
||||||
|
c.Precision = int(datetime.Int64)
|
||||||
|
}
|
||||||
|
c.Collation = collation.String
|
||||||
|
c.Metadata["mysql_type"] = typ
|
||||||
|
c.Metadata["mysql_type_signature"] = typeSignature(c)
|
||||||
|
c.Metadata["mysql_charset"] = charset.String
|
||||||
|
c.Metadata["mysql_datetime_precision"] = datetime.Int64
|
||||||
|
c.Metadata["mysql_default_expression"] = strings.Contains(extra, "DEFAULT_GENERATED") || strings.HasPrefix(strings.ToLower(def.String), "current_timestamp") && (c.Type == "timestamp")
|
||||||
if def.Valid {
|
if def.Valid {
|
||||||
c.Default = def.String
|
c.Default = def.String
|
||||||
|
if strings.Contains(extra, "DEFAULT_GENERATED") {
|
||||||
|
c.Default = unescapeExpression(def.String)
|
||||||
|
}
|
||||||
|
if strings.HasPrefix(typ, "bit(") && def.Valid {
|
||||||
|
c.Metadata["mysql_default_expression"] = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
c.AutoIncrement = strings.Contains(extra, "auto_increment")
|
||||||
|
c.Generated, c.GenerationExpression = generation.String != "", unescapeExpression(generation.String)
|
||||||
|
if strings.Contains(extra, "STORED GENERATED") {
|
||||||
|
c.Metadata["mysql_generation_storage"] = "STORED"
|
||||||
|
} else if c.Generated {
|
||||||
|
c.Metadata["mysql_generation_storage"] = "VIRTUAL"
|
||||||
|
}
|
||||||
|
c.Metadata["mysql_invisible"] = strings.Contains(extra, "INVISIBLE")
|
||||||
|
remaining := onUpdateExtra.ReplaceAllString(strings.ToLower(extra), "")
|
||||||
|
for _, known := range []string{"auto_increment", "default_generated", "stored generated", "virtual generated", "invisible"} {
|
||||||
|
remaining = strings.ReplaceAll(remaining, known, "")
|
||||||
|
}
|
||||||
|
if remaining = strings.TrimSpace(remaining); remaining != "" {
|
||||||
|
c.Metadata["mysql_unsupported_extra"] = remaining
|
||||||
|
}
|
||||||
|
if i := strings.Index(strings.ToLower(extra), "on update "); i >= 0 {
|
||||||
|
c.Metadata["mysql_on_update"] = strings.Fields(extra[i+10:])[0]
|
||||||
|
}
|
||||||
|
if srid.Valid {
|
||||||
|
c.Metadata["mysql_srs_id"] = srid.Int64
|
||||||
}
|
}
|
||||||
c.AutoIncrement = strings.Contains(strings.ToLower(extra), "auto_increment")
|
|
||||||
out[name] = c
|
out[name] = c
|
||||||
}
|
}
|
||||||
return out, rows.Err()
|
if err := rows.Err(); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
var tableName, create string
|
||||||
|
qualified := "`" + strings.ReplaceAll(schema, "`", "``") + "`.`" + strings.ReplaceAll(table.Name, "`", "``") + "`"
|
||||||
|
if err := r.db.QueryRowContext(r.ctx, "SHOW CREATE TABLE "+qualified).Scan(&tableName, &create); err != nil {
|
||||||
|
return nil, fmt.Errorf("native defaults %s.%s: %w", schema, table.Name, err)
|
||||||
|
}
|
||||||
|
// information_schema AUTO_INCREMENT may be stale because of statistics caching.
|
||||||
|
for _, c := range out {
|
||||||
|
if c.AutoIncrement {
|
||||||
|
table.Metadata["mysql_auto_increment"] = autoIncrementValue(create)
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if options := unsupportedNativeOptions(create); len(options) > 0 {
|
||||||
|
table.Metadata["mysql_unsupported_options"] = strings.Join(options, ", ")
|
||||||
|
}
|
||||||
|
for name, sql := range columnDefaults(create) {
|
||||||
|
c := out[name]
|
||||||
|
if c == nil || c.Default == nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if c.Type == "bytea" {
|
||||||
|
c.Default = sql
|
||||||
|
}
|
||||||
|
c.Metadata["mysql_default_sql"] = sql
|
||||||
|
c.Metadata["mysql_default_value"] = fmt.Sprint(c.Default)
|
||||||
|
}
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func typeSignature(c *models.Column) string {
|
||||||
|
return fmt.Sprintf("%s/%d/%d/%d", c.Type, c.Length, c.Precision, c.Scale)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (r *Reader) queryConstraints(schema, table string) (map[string]*models.Constraint, error) {
|
func (r *Reader) queryConstraints(schema, table string) (map[string]*models.Constraint, error) {
|
||||||
rows, err := r.db.QueryContext(r.ctx, `SELECT CONSTRAINT_NAME, CONSTRAINT_TYPE, COLUMN_NAME, REFERENCED_TABLE_SCHEMA, REFERENCED_TABLE_NAME, REFERENCED_COLUMN_NAME, ORDINAL_POSITION FROM information_schema.KEY_COLUMN_USAGE k JOIN information_schema.TABLE_CONSTRAINTS t USING (CONSTRAINT_SCHEMA, TABLE_NAME, CONSTRAINT_NAME) WHERE k.TABLE_SCHEMA=? AND k.TABLE_NAME=? ORDER BY CONSTRAINT_NAME, ORDINAL_POSITION`, schema, table)
|
rows, err := r.db.QueryContext(r.ctx, `SELECT k.CONSTRAINT_NAME, t.CONSTRAINT_TYPE, k.COLUMN_NAME,
|
||||||
|
k.REFERENCED_TABLE_SCHEMA, k.REFERENCED_TABLE_NAME, k.REFERENCED_COLUMN_NAME, k.ORDINAL_POSITION,
|
||||||
|
rc.DELETE_RULE, rc.UPDATE_RULE FROM information_schema.KEY_COLUMN_USAGE k
|
||||||
|
JOIN information_schema.TABLE_CONSTRAINTS t USING (CONSTRAINT_SCHEMA,TABLE_NAME,CONSTRAINT_NAME)
|
||||||
|
LEFT JOIN information_schema.REFERENTIAL_CONSTRAINTS rc ON rc.CONSTRAINT_SCHEMA=k.CONSTRAINT_SCHEMA AND rc.TABLE_NAME=k.TABLE_NAME AND rc.CONSTRAINT_NAME=k.CONSTRAINT_NAME
|
||||||
|
WHERE k.TABLE_SCHEMA=? AND k.TABLE_NAME=? ORDER BY k.CONSTRAINT_NAME,k.ORDINAL_POSITION`, schema, table)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, fmt.Errorf("constraints %s.%s: %w", schema, table, err)
|
||||||
}
|
}
|
||||||
defer rows.Close()
|
|
||||||
out := map[string]*models.Constraint{}
|
out := map[string]*models.Constraint{}
|
||||||
for rows.Next() {
|
for rows.Next() {
|
||||||
var name, typ, col, rs, rt, rc string
|
var name, typ string
|
||||||
|
var col, rs, rt, rc, del, upd sql.NullString
|
||||||
var pos int
|
var pos int
|
||||||
if err := rows.Scan(&name, &typ, &col, &rs, &rt, &rc, &pos); err != nil {
|
if err := rows.Scan(&name, &typ, &col, &rs, &rt, &rc, &pos, &del, &upd); err != nil {
|
||||||
|
rows.Close()
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
c := out[name]
|
c := out[name]
|
||||||
@@ -201,31 +313,54 @@ func (r *Reader) queryConstraints(schema, table string) (map[string]*models.Cons
|
|||||||
ct = models.ForeignKeyConstraint
|
ct = models.ForeignKeyConstraint
|
||||||
}
|
}
|
||||||
c = models.InitConstraint(name, ct)
|
c = models.InitConstraint(name, ct)
|
||||||
c.Schema = schema
|
c.Schema, c.Table = schema, table
|
||||||
c.Table = table
|
c.ReferencedSchema, c.ReferencedTable = rs.String, rt.String
|
||||||
c.ReferencedSchema = rs
|
c.OnDelete, c.OnUpdate = del.String, upd.String
|
||||||
c.ReferencedTable = rt
|
|
||||||
out[name] = c
|
out[name] = c
|
||||||
}
|
}
|
||||||
c.Columns = append(c.Columns, col)
|
if col.Valid {
|
||||||
if rc != "" {
|
c.Columns = append(c.Columns, col.String)
|
||||||
c.ReferencedColumns = append(c.ReferencedColumns, rc)
|
|
||||||
}
|
}
|
||||||
|
if rc.Valid {
|
||||||
|
c.ReferencedColumns = append(c.ReferencedColumns, rc.String)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
err = rows.Err()
|
||||||
|
rows.Close()
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
rows, err = r.db.QueryContext(r.ctx, `SELECT t.CONSTRAINT_NAME,c.CHECK_CLAUSE,t.ENFORCED FROM information_schema.TABLE_CONSTRAINTS t JOIN information_schema.CHECK_CONSTRAINTS c USING (CONSTRAINT_SCHEMA,CONSTRAINT_NAME) WHERE t.TABLE_SCHEMA=? AND t.TABLE_NAME=? AND t.CONSTRAINT_TYPE='CHECK' ORDER BY t.CONSTRAINT_NAME`, schema, table)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("checks %s.%s (requires MySQL 8.0.16+): %w", schema, table, err)
|
||||||
|
}
|
||||||
|
defer rows.Close()
|
||||||
|
for rows.Next() {
|
||||||
|
var name, expr, enforced string
|
||||||
|
if err := rows.Scan(&name, &expr, &enforced); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
c := models.InitConstraint(name, models.CheckConstraint)
|
||||||
|
c.Schema, c.Table, c.Expression = schema, table, unescapeExpression(expr)
|
||||||
|
c.Metadata = map[string]any{"mysql_enforced": enforced == "YES"}
|
||||||
|
out[name] = c
|
||||||
}
|
}
|
||||||
return out, rows.Err()
|
return out, rows.Err()
|
||||||
}
|
}
|
||||||
|
|
||||||
func (r *Reader) queryIndexes(schema, table string) (map[string]*models.Index, error) {
|
func (r *Reader) queryIndexes(schema, table string) (map[string]*models.Index, error) {
|
||||||
rows, err := r.db.QueryContext(r.ctx, `SELECT INDEX_NAME, NON_UNIQUE, COLUMN_NAME, SEQ_IN_INDEX, INDEX_TYPE FROM information_schema.STATISTICS WHERE TABLE_SCHEMA=? AND TABLE_NAME=? AND INDEX_NAME <> 'PRIMARY' ORDER BY INDEX_NAME, SEQ_IN_INDEX`, schema, table)
|
rows, err := r.db.QueryContext(r.ctx, `SELECT INDEX_NAME, NON_UNIQUE, COLUMN_NAME, SEQ_IN_INDEX, INDEX_TYPE, EXPRESSION, SUB_PART, COLLATION, IS_VISIBLE, INDEX_COMMENT FROM information_schema.STATISTICS WHERE TABLE_SCHEMA=? AND TABLE_NAME=? ORDER BY INDEX_NAME,SEQ_IN_INDEX`, schema, table)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, fmt.Errorf("indexes %s.%s (requires MySQL 8.0.13+): %w", schema, table, err)
|
||||||
}
|
}
|
||||||
defer rows.Close()
|
defer rows.Close()
|
||||||
out := map[string]*models.Index{}
|
out := map[string]*models.Index{}
|
||||||
for rows.Next() {
|
for rows.Next() {
|
||||||
var name, col, typ string
|
var name, typ, visible, comment string
|
||||||
|
var col, expr, direction sql.NullString
|
||||||
|
var prefix sql.NullInt64
|
||||||
var non, seq int
|
var non, seq int
|
||||||
if err := rows.Scan(&name, &non, &col, &seq, &typ); err != nil {
|
if err := rows.Scan(&name, &non, &col, &seq, &typ, &expr, &prefix, &direction, &visible, &comment); err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
i := out[name]
|
i := out[name]
|
||||||
@@ -233,22 +368,96 @@ func (r *Reader) queryIndexes(schema, table string) (map[string]*models.Index, e
|
|||||||
i = models.InitIndex(name, table, schema)
|
i = models.InitIndex(name, table, schema)
|
||||||
i.Unique = non == 0
|
i.Unique = non == 0
|
||||||
i.Type = strings.ToLower(typ)
|
i.Type = strings.ToLower(typ)
|
||||||
|
i.Comment = comment
|
||||||
|
i.Metadata["mysql_visible"] = visible == "YES"
|
||||||
out[name] = i
|
out[name] = i
|
||||||
}
|
}
|
||||||
i.Columns = append(i.Columns, col)
|
i.Columns = append(i.Columns, col.String)
|
||||||
|
i.KeyParts = append(i.KeyParts, models.IndexPart{Name: col.String, Expression: unescapeExpression(expr.String), Length: int(prefix.Int64), Descending: direction.String == "D"})
|
||||||
}
|
}
|
||||||
return out, rows.Err()
|
return out, rows.Err()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Record objects outside the supported table-schema contract rather than silently
|
||||||
|
// discarding them. Writers can warn or reject the conversion before any output.
|
||||||
|
func (r *Reader) readUnsupported(db *models.Database, schema string) error {
|
||||||
|
queries := []struct{ kind, sql string }{
|
||||||
|
{"view", `SELECT TABLE_NAME FROM information_schema.VIEWS WHERE TABLE_SCHEMA=?`},
|
||||||
|
{"trigger", `SELECT TRIGGER_NAME FROM information_schema.TRIGGERS WHERE TRIGGER_SCHEMA=?`},
|
||||||
|
{"routine", `SELECT ROUTINE_NAME FROM information_schema.ROUTINES WHERE ROUTINE_SCHEMA=?`},
|
||||||
|
{"event", `SELECT EVENT_NAME FROM information_schema.EVENTS WHERE EVENT_SCHEMA=?`},
|
||||||
|
{"partitioned table", `SELECT DISTINCT TABLE_NAME FROM information_schema.PARTITIONS WHERE TABLE_SCHEMA=? AND PARTITION_NAME IS NOT NULL`},
|
||||||
|
}
|
||||||
|
var unsupported []string
|
||||||
|
for _, q := range queries {
|
||||||
|
rows, err := r.db.QueryContext(r.ctx, q.sql, schema)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("discover %s: %w", q.kind, err)
|
||||||
|
}
|
||||||
|
for rows.Next() {
|
||||||
|
var name string
|
||||||
|
if err := rows.Scan(&name); err != nil {
|
||||||
|
rows.Close()
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
unsupported = append(unsupported, q.kind+" "+schema+"."+name)
|
||||||
|
}
|
||||||
|
err = rows.Err()
|
||||||
|
rows.Close()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(unsupported) > 0 {
|
||||||
|
db.Metadata["mysql_unsupported"] = unsupported
|
||||||
|
for _, s := range db.Schemas {
|
||||||
|
s.Metadata["mysql_unsupported"] = unsupported
|
||||||
|
}
|
||||||
|
if r.options.Progress != nil {
|
||||||
|
for _, item := range unsupported {
|
||||||
|
r.options.Progress("warning: unsupported MySQL " + item)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
func (r *Reader) deriveRelationship(t *models.Table, c *models.Constraint) {
|
func (r *Reader) deriveRelationship(t *models.Table, c *models.Constraint) {
|
||||||
n := fmt.Sprintf("%s_to_%s", t.Name, c.ReferencedTable)
|
n := fmt.Sprintf("%s_%s", t.Name, c.Name)
|
||||||
rel := models.InitRelationship(n, models.OneToMany)
|
rel := models.InitRelationship(n, models.OneToMany)
|
||||||
rel.FromTable = t.Name
|
rel.FromTable, rel.FromSchema = t.Name, t.Schema
|
||||||
rel.FromSchema = t.Schema
|
|
||||||
rel.FromColumns = append([]string(nil), c.Columns...)
|
rel.FromColumns = append([]string(nil), c.Columns...)
|
||||||
rel.ToTable = c.ReferencedTable
|
rel.ToTable, rel.ToSchema = c.ReferencedTable, c.ReferencedSchema
|
||||||
rel.ToSchema = c.ReferencedSchema
|
|
||||||
rel.ToColumns = append([]string(nil), c.ReferencedColumns...)
|
rel.ToColumns = append([]string(nil), c.ReferencedColumns...)
|
||||||
rel.ForeignKey = c.Name
|
rel.ForeignKey = c.Name
|
||||||
t.Relationships[n] = rel
|
t.Relationships[n] = rel
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// MySQL information_schema escapes SQL expressions by an additional layer
|
||||||
|
// (including quote delimiters). This covers defaults, generated columns, checks,
|
||||||
|
// and functional indexes. Literal defaults are already values and bypass this.
|
||||||
|
func unescapeExpression(s string) string {
|
||||||
|
var out strings.Builder
|
||||||
|
for i := 0; i < len(s); i++ {
|
||||||
|
if s[i] == '\\' && i+1 < len(s) {
|
||||||
|
i++
|
||||||
|
switch s[i] {
|
||||||
|
case 'n':
|
||||||
|
out.WriteByte('\n')
|
||||||
|
case 'r':
|
||||||
|
out.WriteByte('\r')
|
||||||
|
case 't':
|
||||||
|
out.WriteByte('\t')
|
||||||
|
case '0':
|
||||||
|
out.WriteByte(0)
|
||||||
|
case 'Z':
|
||||||
|
out.WriteByte(26)
|
||||||
|
default:
|
||||||
|
out.WriteByte(s[i])
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
out.WriteByte(s[i])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out.String()
|
||||||
|
}
|
||||||
|
|||||||
@@ -3,12 +3,13 @@ package mysql
|
|||||||
import (
|
import (
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
|
"git.warky.dev/wdevs/relspecgo/pkg/models"
|
||||||
"git.warky.dev/wdevs/relspecgo/pkg/readers"
|
"git.warky.dev/wdevs/relspecgo/pkg/readers"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestReaderMapDataType(t *testing.T) {
|
func TestReaderMapDataType(t *testing.T) {
|
||||||
r := NewReader(&readers.ReaderOptions{})
|
r := NewReader(&readers.ReaderOptions{})
|
||||||
for _, tc := range []struct{ input, want string }{{"varchar(64)", "string"}, {"bigint unsigned", "int64"}, {"datetime", "timestamp"}, {"json", "json"}} {
|
for _, tc := range []struct{ input, want string }{{"varchar(64)", "string"}, {"bigint unsigned", "uint64"}, {"datetime", "timestamp"}, {"json", "json"}} {
|
||||||
if got := r.mapDataType(tc.input); got != tc.want {
|
if got := r.mapDataType(tc.input); got != tc.want {
|
||||||
t.Errorf("mapDataType(%q) = %q, want %q", tc.input, got, tc.want)
|
t.Errorf("mapDataType(%q) = %q, want %q", tc.input, got, tc.want)
|
||||||
}
|
}
|
||||||
@@ -20,3 +21,17 @@ func TestReaderRequiresConnectionString(t *testing.T) {
|
|||||||
t.Fatal("expected missing connection string error")
|
t.Fatal("expected missing connection string error")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestRelationshipNamesCannotCollapseDistinctForeignKeys(t *testing.T) {
|
||||||
|
table := models.InitTable("t", "app")
|
||||||
|
reader := NewReader(&readers.ReaderOptions{})
|
||||||
|
for _, c := range []*models.Constraint{
|
||||||
|
{Name: "a_to_b", ReferencedTable: "c"},
|
||||||
|
{Name: "a", ReferencedTable: "b_to_c"},
|
||||||
|
} {
|
||||||
|
reader.deriveRelationship(table, c)
|
||||||
|
}
|
||||||
|
if len(table.Relationships) != 2 {
|
||||||
|
t.Fatalf("distinct foreign keys collapsed: %+v", table.Relationships)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -152,7 +152,7 @@ func (r *Reader) readScripts() ([]*models.Script, error) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("failed to read file %s: %w", path, err)
|
return fmt.Errorf("failed to read file %s: %w", path, err)
|
||||||
}
|
}
|
||||||
sql, err := assetloader.ProcessEmbedDirectives(path, string(content))
|
sql, err := assetloader.ProcessEmbedDirectivesWithOptions(path, string(content), assetloader.EmbedOptions{AssetRoot: r.options.FilePath})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,136 @@
|
|||||||
|
// Package safefile provides portable path validation and root-scoped file access.
|
||||||
|
package safefile
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"unicode"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Relative validates artifact paths consistently on Unix and Windows. It rejects
|
||||||
|
// traversal rather than normalizing it, so aliases cannot hide filename collisions.
|
||||||
|
func Relative(name string) (string, error) {
|
||||||
|
if name == "" || !filepath.IsLocal(name) || strings.ContainsAny(name, `\:<>"|?*`) {
|
||||||
|
return "", fmt.Errorf("unsafe relative path %q", name)
|
||||||
|
}
|
||||||
|
for _, r := range name {
|
||||||
|
if unicode.IsControl(r) {
|
||||||
|
return "", fmt.Errorf("control character in path %q", name)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, part := range strings.Split(filepath.ToSlash(name), "/") {
|
||||||
|
if part == "" || part == "." || part == ".." || strings.HasSuffix(part, ".") || strings.HasSuffix(part, " ") {
|
||||||
|
return "", fmt.Errorf("unsafe path component in %q", name)
|
||||||
|
}
|
||||||
|
base := strings.ToUpper(strings.SplitN(part, ".", 2)[0])
|
||||||
|
device := base == "CON" || base == "PRN" || base == "AUX" || base == "NUL" || base == "CONIN$" || base == "CONOUT$"
|
||||||
|
for _, prefix := range []string{"COM", "LPT"} {
|
||||||
|
if strings.HasPrefix(base, prefix) {
|
||||||
|
suffix := []rune(strings.TrimPrefix(base, prefix))
|
||||||
|
device = device || (len(suffix) == 1 && strings.ContainsRune("123456789¹²³", suffix[0]))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if device {
|
||||||
|
return "", fmt.Errorf("reserved device path %q", name)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return filepath.FromSlash(name), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Check rejects existing symlinks and special files; os.Root separately enforces
|
||||||
|
// containment at open/rename time, including if a symlink changes after this check.
|
||||||
|
func Check(root *os.Root, name string) error {
|
||||||
|
if _, err := Relative(filepath.ToSlash(name)); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
parts := strings.Split(name, string(filepath.Separator))
|
||||||
|
current := ""
|
||||||
|
for i, part := range parts {
|
||||||
|
current = filepath.Join(current, part)
|
||||||
|
info, err := root.Lstat(current)
|
||||||
|
if os.IsNotExist(err) {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if info.Mode()&os.ModeSymlink != 0 {
|
||||||
|
return fmt.Errorf("symlink is not allowed in path %q", name)
|
||||||
|
}
|
||||||
|
if i < len(parts)-1 && !info.IsDir() {
|
||||||
|
return fmt.Errorf("parent is not a directory in path %q", name)
|
||||||
|
}
|
||||||
|
if i == len(parts)-1 && !info.Mode().IsRegular() {
|
||||||
|
return fmt.Errorf("path is not a regular file: %q", name)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Read opens through a pinned root and bounds both the declared and actual size.
|
||||||
|
func Read(root *os.Root, name string, maxBytes int64) ([]byte, error) {
|
||||||
|
if maxBytes <= 0 || maxBytes == (1<<63)-1 {
|
||||||
|
return nil, fmt.Errorf("file size limit must be positive and bounded")
|
||||||
|
}
|
||||||
|
if err := Check(root, name); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
file, err := root.Open(name)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
defer file.Close()
|
||||||
|
info, err := file.Stat()
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if !info.Mode().IsRegular() {
|
||||||
|
return nil, fmt.Errorf("asset is not a regular file: %q", name)
|
||||||
|
}
|
||||||
|
if info.Size() > maxBytes {
|
||||||
|
return nil, fmt.Errorf("asset %q exceeds %d byte limit", name, maxBytes)
|
||||||
|
}
|
||||||
|
data, err := io.ReadAll(io.LimitReader(file, maxBytes+1))
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if int64(len(data)) > maxBytes {
|
||||||
|
return nil, fmt.Errorf("asset %q exceeds %d byte limit", name, maxBytes)
|
||||||
|
}
|
||||||
|
return data, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// CheckRoot rejects symlinks in the configured root or its existing ancestors.
|
||||||
|
// The configured root is a trusted caller-selected boundary; OpenRoot pins it for
|
||||||
|
// subsequent operations. Missing components are allowed for output root creation.
|
||||||
|
func CheckRoot(path string) error {
|
||||||
|
absolute, err := filepath.Abs(path)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
volume := filepath.VolumeName(absolute)
|
||||||
|
current := volume + string(filepath.Separator)
|
||||||
|
for _, part := range strings.Split(strings.TrimPrefix(absolute, current), string(filepath.Separator)) {
|
||||||
|
if part == "" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
current = filepath.Join(current, part)
|
||||||
|
info, err := os.Lstat(current)
|
||||||
|
if os.IsNotExist(err) {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if info.Mode()&os.ModeSymlink != 0 {
|
||||||
|
return fmt.Errorf("symlink is not allowed in root path %q", path)
|
||||||
|
}
|
||||||
|
if !info.IsDir() {
|
||||||
|
return fmt.Errorf("root parent is not a directory: %q", current)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
@@ -0,0 +1,101 @@
|
|||||||
|
// Package sqllex recognizes PostgreSQL lexical boundaries without parsing SQL.
|
||||||
|
package sqllex
|
||||||
|
|
||||||
|
import "strings"
|
||||||
|
|
||||||
|
func start(c byte) bool { return c >= 128 || c == '_' || c >= 'a' && c <= 'z' || c >= 'A' && c <= 'Z' }
|
||||||
|
func ident(c byte) bool { return start(c) || c >= '0' && c <= '9' || c >= 128 }
|
||||||
|
|
||||||
|
// Walk visits named parameters and standalone line comments in executable SQL. Quoted
|
||||||
|
// strings/identifiers, dollar bodies, casts and nested block comments are opaque.
|
||||||
|
// Returning replacement text is literal: it is never scanned again.
|
||||||
|
func Walk(sql string, parameter, lineComment func(string) string) string {
|
||||||
|
var out strings.Builder
|
||||||
|
for i := 0; i < len(sql); {
|
||||||
|
begin := i
|
||||||
|
switch {
|
||||||
|
case strings.HasPrefix(sql[i:], "--"):
|
||||||
|
i += 2
|
||||||
|
for i < len(sql) && sql[i] != '\n' && sql[i] != '\r' {
|
||||||
|
i++
|
||||||
|
}
|
||||||
|
text := sql[begin:i]
|
||||||
|
if lineComment != nil && strings.TrimSpace(sql[strings.LastIndexAny(sql[:begin], "\r\n")+1:begin]) == "" {
|
||||||
|
text = lineComment(text)
|
||||||
|
}
|
||||||
|
out.WriteString(text)
|
||||||
|
continue
|
||||||
|
case strings.HasPrefix(sql[i:], "/*"):
|
||||||
|
i += 2
|
||||||
|
depth := 1
|
||||||
|
for i < len(sql) && depth > 0 {
|
||||||
|
if strings.HasPrefix(sql[i:], "/*") {
|
||||||
|
depth++
|
||||||
|
i += 2
|
||||||
|
} else if strings.HasPrefix(sql[i:], "*/") {
|
||||||
|
depth--
|
||||||
|
i += 2
|
||||||
|
} else {
|
||||||
|
i++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
case sql[i] == '\'' || sql[i] == '"':
|
||||||
|
q := sql[i]
|
||||||
|
escapes := q == '\'' && i > 0 && (sql[i-1] == 'E' || sql[i-1] == 'e') && (i == 1 || !ident(sql[i-2]))
|
||||||
|
i++
|
||||||
|
for i < len(sql) {
|
||||||
|
if escapes && sql[i] == '\\' {
|
||||||
|
i++
|
||||||
|
if i < len(sql) {
|
||||||
|
i++
|
||||||
|
}
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if sql[i] == q {
|
||||||
|
i++
|
||||||
|
if i < len(sql) && sql[i] == q {
|
||||||
|
i++
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
break
|
||||||
|
}
|
||||||
|
i++
|
||||||
|
}
|
||||||
|
case sql[i] == '$' && (i == 0 || !ident(sql[i-1]) && sql[i-1] != '$'):
|
||||||
|
j := i + 1
|
||||||
|
if j < len(sql) && start(sql[j]) {
|
||||||
|
for j < len(sql) && ident(sql[j]) {
|
||||||
|
j++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if j < len(sql) && sql[j] == '$' {
|
||||||
|
tag := sql[i : j+1]
|
||||||
|
end := strings.Index(sql[j+1:], tag)
|
||||||
|
if end < 0 {
|
||||||
|
i = len(sql)
|
||||||
|
} else {
|
||||||
|
i = j + 1 + end + len(tag)
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
i++
|
||||||
|
}
|
||||||
|
case strings.HasPrefix(sql[i:], "::"):
|
||||||
|
i += 2
|
||||||
|
case sql[i] == ':' && i+1 < len(sql) && start(sql[i+1]) && (i == 0 || !ident(sql[i-1]) && sql[i-1] != ':'):
|
||||||
|
i += 2
|
||||||
|
for i < len(sql) && ident(sql[i]) {
|
||||||
|
i++
|
||||||
|
}
|
||||||
|
text := sql[begin:i]
|
||||||
|
if parameter != nil {
|
||||||
|
text = parameter(text)
|
||||||
|
}
|
||||||
|
out.WriteString(text)
|
||||||
|
continue
|
||||||
|
default:
|
||||||
|
i++
|
||||||
|
}
|
||||||
|
out.WriteString(sql[begin:i])
|
||||||
|
}
|
||||||
|
return out.String()
|
||||||
|
}
|
||||||
@@ -0,0 +1,28 @@
|
|||||||
|
package sqllex
|
||||||
|
|
||||||
|
import (
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestWalkSQLContexts(t *testing.T) {
|
||||||
|
sql := `SELECT :x,:x, :x::text, ':hidden', 'it'':hidden', E'it\':hidden', ":hidden", $$:hidden$$, $名$:hidden$名$;
|
||||||
|
-- :hidden
|
||||||
|
/* :hidden /* :hidden */ :hidden */ SELECT :y, value::integer, x := 1;`
|
||||||
|
var names []string
|
||||||
|
got := Walk(sql, func(name string) string { names = append(names, name); return "$1 ${2} :inserted" }, nil)
|
||||||
|
if strings.Join(names, ",") != ":x,:x,:x,:y" {
|
||||||
|
t.Fatalf("parameters: %v", names)
|
||||||
|
}
|
||||||
|
if strings.Count(got, "$1 ${2} :inserted") != 4 || !strings.Contains(got, "value::integer") {
|
||||||
|
t.Fatalf("result: %s", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestUnterminatedSQLContextsAreOpaque(t *testing.T) {
|
||||||
|
for _, sql := range []string{"SELECT ':x", `SELECT ":x`, "DO $tag$:x", "/* :x", "-- :x"} {
|
||||||
|
if got := Walk(sql, func(string) string { t.Fatal("unexpected parameter"); return "" }, nil); got != sql {
|
||||||
|
t.Fatalf("changed %q", sql)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,157 @@
|
|||||||
|
package transform
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"reflect"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"git.warky.dev/wdevs/relspecgo/pkg/models"
|
||||||
|
)
|
||||||
|
|
||||||
|
// LowercaseIdentifiers copies the model and lowercases names and structured
|
||||||
|
// references. SQL expressions, defaults, enum values, comments and metadata are
|
||||||
|
// preserved. Callers must adjust identifiers inside their own SQL expressions.
|
||||||
|
// Collisions are rejected before any output is written.
|
||||||
|
func LowercaseIdentifiers(db *models.Database) (*models.Database, error) {
|
||||||
|
if db == nil {
|
||||||
|
return nil, fmt.Errorf("database is required")
|
||||||
|
}
|
||||||
|
seen := map[any]reflect.Value{}
|
||||||
|
nameFields := map[string]bool{}
|
||||||
|
for _, name := range strings.Fields("Name Schema Table Tablespace Collation TableName SchemaName FromTable FromSchema ToTable ToSchema ThroughTable ThroughSchema ForeignKey ReferencedTable ReferencedSchema OwnedByTable OwnedByColumn") {
|
||||||
|
nameFields[name] = true
|
||||||
|
}
|
||||||
|
listFields := map[string]bool{"Columns": true, "Include": true, "ReferencedColumns": true, "FromColumns": true, "ToColumns": true, "RunAfter": true}
|
||||||
|
var copyValue func(reflect.Value) (reflect.Value, error)
|
||||||
|
copyValue = func(v reflect.Value) (reflect.Value, error) {
|
||||||
|
if !v.IsValid() {
|
||||||
|
return v, nil
|
||||||
|
}
|
||||||
|
switch v.Kind() {
|
||||||
|
case reflect.Pointer:
|
||||||
|
if v.IsNil() || v.Type().Elem().PkgPath() != reflect.TypeOf(models.Database{}).PkgPath() {
|
||||||
|
return v, nil
|
||||||
|
}
|
||||||
|
if prior, ok := seen[v.Interface()]; ok {
|
||||||
|
return prior, nil
|
||||||
|
}
|
||||||
|
out := reflect.New(v.Type().Elem())
|
||||||
|
seen[v.Interface()] = out
|
||||||
|
copied, err := copyValue(v.Elem())
|
||||||
|
if err != nil {
|
||||||
|
return reflect.Value{}, err
|
||||||
|
}
|
||||||
|
out.Elem().Set(copied)
|
||||||
|
return out, nil
|
||||||
|
case reflect.Struct:
|
||||||
|
out := reflect.New(v.Type()).Elem()
|
||||||
|
out.Set(v)
|
||||||
|
for i := 0; i < v.NumField(); i++ {
|
||||||
|
field := v.Type().Field(i).Name
|
||||||
|
value := v.Field(i)
|
||||||
|
if nameFields[field] && value.Kind() == reflect.String {
|
||||||
|
out.Field(i).SetString(strings.ToLower(value.String()))
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if (v.Type() != reflect.TypeOf(models.Index{}) || field != "Columns") && listFields[field] && value.Kind() == reflect.Slice && value.Type().Elem().Kind() == reflect.String {
|
||||||
|
list := reflect.MakeSlice(value.Type(), value.Len(), value.Len())
|
||||||
|
for j := 0; j < value.Len(); j++ {
|
||||||
|
list.Index(j).SetString(strings.ToLower(value.Index(j).String()))
|
||||||
|
}
|
||||||
|
out.Field(i).Set(list)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if field == "Metadata" || field == "Default" || field == "Properties" || field == "Permissions" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
copied, err := copyValue(value)
|
||||||
|
if err != nil {
|
||||||
|
return reflect.Value{}, err
|
||||||
|
}
|
||||||
|
out.Field(i).Set(copied)
|
||||||
|
}
|
||||||
|
if v.Type() == reflect.TypeOf(models.Table{}) {
|
||||||
|
original := v.Interface().(models.Table)
|
||||||
|
copied := out.Addr().Interface().(*models.Table)
|
||||||
|
for originalKey, index := range original.Indexes {
|
||||||
|
if index == nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
target := copied.Indexes[strings.ToLower(originalKey)]
|
||||||
|
target.Columns = append([]string(nil), index.Columns...)
|
||||||
|
for i, reference := range index.Columns {
|
||||||
|
for _, column := range original.Columns {
|
||||||
|
if column != nil && (reference == column.Name || reference == `"`+strings.ReplaceAll(column.Name, `"`, `""`)+`"`) {
|
||||||
|
target.Columns[i] = strings.ToLower(column.Name)
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out, nil
|
||||||
|
case reflect.Slice:
|
||||||
|
if v.IsNil() {
|
||||||
|
return v, nil
|
||||||
|
}
|
||||||
|
out := reflect.MakeSlice(v.Type(), v.Len(), v.Len())
|
||||||
|
names := map[string]bool{}
|
||||||
|
for i := 0; i < v.Len(); i++ {
|
||||||
|
copied, err := copyValue(v.Index(i))
|
||||||
|
if err != nil {
|
||||||
|
return reflect.Value{}, err
|
||||||
|
}
|
||||||
|
out.Index(i).Set(copied)
|
||||||
|
if copied.Kind() == reflect.Pointer && !copied.IsNil() && copied.Elem().Kind() == reflect.Struct {
|
||||||
|
name := copied.Elem().FieldByName("Name")
|
||||||
|
if name.IsValid() {
|
||||||
|
n := name.String()
|
||||||
|
if names[n] {
|
||||||
|
return reflect.Value{}, fmt.Errorf("lowercase identifier collision: %q", n)
|
||||||
|
}
|
||||||
|
names[n] = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out, nil
|
||||||
|
case reflect.Map:
|
||||||
|
if v.IsNil() || v.Type().Elem().Kind() != reflect.Pointer {
|
||||||
|
return v, nil
|
||||||
|
}
|
||||||
|
out := reflect.MakeMap(v.Type())
|
||||||
|
names := map[string]bool{}
|
||||||
|
iter := v.MapRange()
|
||||||
|
for iter.Next() {
|
||||||
|
key := iter.Key()
|
||||||
|
if key.Kind() == reflect.String {
|
||||||
|
key = reflect.ValueOf(strings.ToLower(key.String())).Convert(key.Type())
|
||||||
|
}
|
||||||
|
if out.MapIndex(key).IsValid() {
|
||||||
|
return reflect.Value{}, fmt.Errorf("lowercase identifier collision: %v", key.Interface())
|
||||||
|
}
|
||||||
|
copied, err := copyValue(iter.Value())
|
||||||
|
if err != nil {
|
||||||
|
return reflect.Value{}, err
|
||||||
|
}
|
||||||
|
if copied.Kind() == reflect.Pointer && !copied.IsNil() && copied.Elem().Kind() == reflect.Struct {
|
||||||
|
name := copied.Elem().FieldByName("Name")
|
||||||
|
if name.IsValid() {
|
||||||
|
n := name.String()
|
||||||
|
if names[n] {
|
||||||
|
return reflect.Value{}, fmt.Errorf("lowercase identifier collision: %q", n)
|
||||||
|
}
|
||||||
|
names[n] = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
out.SetMapIndex(key, copied)
|
||||||
|
}
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
return v, nil
|
||||||
|
}
|
||||||
|
value, err := copyValue(reflect.ValueOf(db))
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return value.Interface().(*models.Database), nil
|
||||||
|
}
|
||||||
@@ -0,0 +1,53 @@
|
|||||||
|
package transform
|
||||||
|
|
||||||
|
import (
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"git.warky.dev/wdevs/relspecgo/pkg/models"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestLowercaseIdentifiersReferencesAndData(t *testing.T) {
|
||||||
|
db := models.InitDatabase("DB")
|
||||||
|
s := models.InitSchema("App")
|
||||||
|
tbl := models.InitTable("Users", "App")
|
||||||
|
col := models.InitColumn("ID", "Users", "App")
|
||||||
|
tbl.Tablespace = "FastStore"
|
||||||
|
col.Collation = "MyCollation"
|
||||||
|
col.Default = "Keep CASE"
|
||||||
|
col.GenerationExpression = `"ID" + 1`
|
||||||
|
tbl.Columns[col.Name] = col
|
||||||
|
tbl.Constraints["FK"] = &models.Constraint{Name: "FK", Columns: []string{"ID"}, ReferencedSchema: "Other", ReferencedTable: "Target", ReferencedColumns: []string{"Key"}}
|
||||||
|
tbl.Indexes["IDX"] = &models.Index{Name: "IDX", Columns: []string{"ID", "lower('VALUE')"}, Include: []string{"ID"}, Where: `"ID" > 0`}
|
||||||
|
tbl.RefSchema = s
|
||||||
|
s.Tables = append(s.Tables, tbl)
|
||||||
|
db.Schemas = append(db.Schemas, s)
|
||||||
|
got, err := LowercaseIdentifiers(db)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
table := got.Schemas[0].Tables[0]
|
||||||
|
if table.Tablespace != "faststore" || table.Columns["id"].Collation != "mycollation" || got.Name != "db" || table.Name != "users" || table.Schema != "app" || table.Columns["id"].Name != "id" || table.Constraints["fk"].ReferencedColumns[0] != "key" || table.Indexes["idx"].Include[0] != "id" || table.RefSchema != got.Schemas[0] {
|
||||||
|
t.Fatalf("bad normalized model: %+v", table)
|
||||||
|
}
|
||||||
|
if table.Columns["id"].Default != "Keep CASE" || table.Columns["id"].GenerationExpression != `"ID" + 1` || table.Indexes["idx"].Where != `"ID" > 0` || table.Indexes["idx"].Columns[1] != "lower('VALUE')" {
|
||||||
|
t.Fatal("data/expression changed")
|
||||||
|
}
|
||||||
|
if tbl.Name != "Users" || tbl.Columns["ID"] != col {
|
||||||
|
t.Fatal("input mutated")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestLowercaseIdentifierCollisions(t *testing.T) {
|
||||||
|
db := models.InitDatabase("db")
|
||||||
|
s := models.InitSchema("app")
|
||||||
|
db.Schemas = append(db.Schemas, s)
|
||||||
|
s.Tables = []*models.Table{models.InitTable("Users", "app"), models.InitTable("users", "app")}
|
||||||
|
if _, err := LowercaseIdentifiers(db); err == nil {
|
||||||
|
t.Fatal("expected table collision")
|
||||||
|
}
|
||||||
|
s.Tables = s.Tables[:1]
|
||||||
|
s.Tables[0].Columns = map[string]*models.Column{"ID": {Name: "ID"}, "id": {Name: "id"}}
|
||||||
|
if _, err := LowercaseIdentifiers(db); err == nil {
|
||||||
|
t.Fatal("expected column collision")
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,239 @@
|
|||||||
|
package writers
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
// ExecutionReport describes an entire invocation, including operations never attempted.
|
||||||
|
// ExecutedStatements counts operations still applied, excluding confirmed rollbacks.
|
||||||
|
type ExecutionReport struct {
|
||||||
|
Dialect string `json:"dialect,omitempty"`
|
||||||
|
TotalStatements int `json:"total_statements"`
|
||||||
|
ExecutedStatements int `json:"executed_statements"`
|
||||||
|
FailedStatements int `json:"failed_statements"`
|
||||||
|
SkippedStatements int `json:"skipped_statements"`
|
||||||
|
RolledBackStatements int `json:"rolled_back_statements"`
|
||||||
|
Outcome string `json:"outcome"`
|
||||||
|
Operations []ExecutionOperation `json:"operations"`
|
||||||
|
Schemas []SchemaReport `json:"schemas"`
|
||||||
|
Errors []ExecutionError `json:"errors,omitempty"`
|
||||||
|
StartTime string `json:"start_time"`
|
||||||
|
EndTime string `json:"end_time"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type SchemaReport struct {
|
||||||
|
Name string `json:"name"`
|
||||||
|
Tables []TableReport `json:"tables"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type TableReport struct {
|
||||||
|
Name string `json:"name"`
|
||||||
|
Created bool `json:"created"`
|
||||||
|
Error string `json:"error,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type ExecutionError struct {
|
||||||
|
StatementNumber int `json:"statement_number"`
|
||||||
|
Statement string `json:"statement"`
|
||||||
|
Error string `json:"error"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// ExecutionOperation is one generated statement or supplied script, in execution order.
|
||||||
|
// A failed raw script may have committed some of its own statements; see Outcome.
|
||||||
|
type ExecutionOperation struct {
|
||||||
|
StatementNumber int `json:"statement_number"`
|
||||||
|
Name string `json:"name,omitempty"`
|
||||||
|
Statement string `json:"statement"`
|
||||||
|
Status string `json:"status"` // applied, executed (raw script), failed, skipped, rolled_back, unknown
|
||||||
|
Error string `json:"error,omitempty"`
|
||||||
|
Skip bool `json:"-"`
|
||||||
|
StopOnFailure bool `json:"-"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// ExecutionFailure is returned even when continuation was explicitly requested.
|
||||||
|
type ExecutionFailure struct{ Report *ExecutionReport }
|
||||||
|
|
||||||
|
func (e *ExecutionFailure) Error() string {
|
||||||
|
message := fmt.Sprintf("%s execution failed: %d failed, %d executed, %d skipped, %d rolled back",
|
||||||
|
e.Report.Dialect, e.Report.FailedStatements, e.Report.ExecutedStatements,
|
||||||
|
e.Report.SkippedStatements, e.Report.RolledBackStatements)
|
||||||
|
if len(e.Report.Errors) > 0 {
|
||||||
|
first := e.Report.Errors[0]
|
||||||
|
message += fmt.Sprintf("; statement %d: %s", first.StatementNumber, first.Error)
|
||||||
|
}
|
||||||
|
return message
|
||||||
|
}
|
||||||
|
|
||||||
|
// ExecutionBackend operates on one connection. Begin switches Exec to a transaction.
|
||||||
|
// Atomic execution stops at the first failure and rolls back the whole invocation.
|
||||||
|
type ExecutionBackend struct {
|
||||||
|
Exec func(context.Context, int, string) error
|
||||||
|
Begin func(context.Context) error
|
||||||
|
Commit func(context.Context) error
|
||||||
|
Rollback func(context.Context) error
|
||||||
|
// ScriptManaged means arbitrary scripts can commit/rollback internally.
|
||||||
|
ScriptManaged bool
|
||||||
|
}
|
||||||
|
|
||||||
|
func (o *WriterOptions) ShouldContinue() bool {
|
||||||
|
legacy, _ := o.Metadata["ignore_errors"].(bool)
|
||||||
|
return o.ContinueOnError || legacy
|
||||||
|
}
|
||||||
|
|
||||||
|
// RunExecution applies a plan, records every outcome, and publishes metadata and reports.
|
||||||
|
func RunExecution(ctx context.Context, opts *WriterOptions, report *ExecutionReport, operations []ExecutionOperation, backend ExecutionBackend) error {
|
||||||
|
report.TotalStatements = len(operations)
|
||||||
|
report.Operations = append([]ExecutionOperation(nil), operations...)
|
||||||
|
report.ExecutedStatements, report.FailedStatements, report.SkippedStatements, report.RolledBackStatements = 0, 0, 0, 0
|
||||||
|
report.Errors = nil
|
||||||
|
report.StartTime = time.Now().Format(time.RFC3339Nano)
|
||||||
|
report.Outcome = "not_started"
|
||||||
|
for i := range report.Operations {
|
||||||
|
report.Operations[i].StatementNumber = i + 1
|
||||||
|
report.Operations[i].Status = "skipped"
|
||||||
|
}
|
||||||
|
var runErr error
|
||||||
|
atomic := opts.AtomicExecution
|
||||||
|
if atomic && opts.ShouldContinue() {
|
||||||
|
runErr = errors.New("atomic execution cannot be combined with continue-on-error")
|
||||||
|
} else if atomic && (backend.Begin == nil || backend.Commit == nil || backend.Rollback == nil) {
|
||||||
|
runErr = errors.New("atomic execution is not supported by this executor")
|
||||||
|
} else if atomic {
|
||||||
|
runErr = backend.Begin(ctx)
|
||||||
|
}
|
||||||
|
if runErr == nil {
|
||||||
|
report.Outcome = "nontransactional"
|
||||||
|
if backend.ScriptManaged {
|
||||||
|
report.Outcome = "script_managed"
|
||||||
|
}
|
||||||
|
stopped := false
|
||||||
|
for i := range report.Operations {
|
||||||
|
op := &report.Operations[i]
|
||||||
|
if stopped || op.Skip || strings.TrimSpace(op.Statement) == "" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if err := backend.Exec(ctx, i, op.Statement); err != nil {
|
||||||
|
runErr = errors.Join(runErr, err)
|
||||||
|
op.Status, op.Error = "failed", err.Error()
|
||||||
|
report.FailedStatements++
|
||||||
|
report.Errors = append(report.Errors, ExecutionError{i + 1, op.Statement, err.Error()})
|
||||||
|
stopped = op.StopOnFailure || !opts.ShouldContinue()
|
||||||
|
} else {
|
||||||
|
op.Status = "applied"
|
||||||
|
if backend.ScriptManaged {
|
||||||
|
op.Status = "executed"
|
||||||
|
}
|
||||||
|
report.ExecutedStatements++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if atomic {
|
||||||
|
if report.FailedStatements > 0 {
|
||||||
|
// Cleanup uses a bounded context independent of caller cancellation.
|
||||||
|
cleanup, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||||
|
rollbackErr := backend.Rollback(cleanup)
|
||||||
|
cancel()
|
||||||
|
if rollbackErr == nil {
|
||||||
|
report.Outcome = "rolled_back"
|
||||||
|
for i := range report.Operations {
|
||||||
|
if report.Operations[i].Status == "applied" {
|
||||||
|
report.Operations[i].Status = "rolled_back"
|
||||||
|
report.RolledBackStatements++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
report.ExecutedStatements = 0
|
||||||
|
} else {
|
||||||
|
report.Outcome = "unknown"
|
||||||
|
runErr = errors.Join(runErr, fmt.Errorf("rollback failed: %w", rollbackErr))
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
runErr = backend.Commit(ctx)
|
||||||
|
if runErr == nil {
|
||||||
|
report.Outcome = "committed"
|
||||||
|
} else {
|
||||||
|
report.Outcome = "unknown"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if report.Outcome == "rolled_back" || report.Outcome == "unknown" {
|
||||||
|
for i := range report.Schemas {
|
||||||
|
for j := range report.Schemas[i].Tables {
|
||||||
|
report.Schemas[i].Tables[j].Created = false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if report.Outcome == "unknown" {
|
||||||
|
for i := range report.Operations {
|
||||||
|
if report.Operations[i].Status == "applied" {
|
||||||
|
report.Operations[i].Status = "unknown"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
report.ExecutedStatements = 0
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, op := range report.Operations {
|
||||||
|
if op.Status == "skipped" {
|
||||||
|
report.SkippedStatements++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
report.EndTime = time.Now().Format(time.RFC3339Nano)
|
||||||
|
if report.FailedStatements > 0 {
|
||||||
|
runErr = errors.Join(&ExecutionFailure{report}, runErr)
|
||||||
|
}
|
||||||
|
return FinishExecution(opts, report, runErr)
|
||||||
|
}
|
||||||
|
|
||||||
|
// FinishExecution publishes legacy counters and a typed report even on failure.
|
||||||
|
func FinishExecution(opts *WriterOptions, report *ExecutionReport, runErr error) error {
|
||||||
|
if opts.Metadata == nil {
|
||||||
|
opts.Metadata = map[string]interface{}{}
|
||||||
|
}
|
||||||
|
opts.Metadata["execution_report"] = report
|
||||||
|
opts.Metadata["execution_total"] = report.TotalStatements
|
||||||
|
opts.Metadata["execution_success"] = report.ExecutedStatements
|
||||||
|
opts.Metadata["execution_failed"] = report.FailedStatements
|
||||||
|
opts.Metadata["execution_skipped"] = report.SkippedStatements
|
||||||
|
opts.Metadata["execution_rolled_back"] = report.RolledBackStatements
|
||||||
|
if path, _ := opts.Metadata["report_path"].(string); path != "" {
|
||||||
|
if err := WriteExecutionReport(path, report); err != nil {
|
||||||
|
runErr = errors.Join(runErr, fmt.Errorf("failed to write requested execution report: %w", err))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return runErr
|
||||||
|
}
|
||||||
|
|
||||||
|
func WriteExecutionReport(path string, report *ExecutionReport) error {
|
||||||
|
file, err := os.Create(path)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("failed to create report file: %w", err)
|
||||||
|
}
|
||||||
|
encoder := json.NewEncoder(file)
|
||||||
|
encoder.SetIndent("", " ")
|
||||||
|
return errors.Join(encoder.Encode(report), file.Close())
|
||||||
|
}
|
||||||
|
|
||||||
|
// StatementOperations creates a plan from the writers' structured statement lists.
|
||||||
|
func StatementOperations(statements []string) []ExecutionOperation {
|
||||||
|
ops := make([]ExecutionOperation, len(statements))
|
||||||
|
for i, stmt := range statements {
|
||||||
|
ops[i] = ExecutionOperation{Statement: stmt, StopOnFailure: strings.HasPrefix(strings.TrimSpace(stmt), "DO $relspec_preflight"), Skip: strings.TrimSpace(stmt) == "" || strings.HasPrefix(strings.TrimSpace(stmt), "--")}
|
||||||
|
}
|
||||||
|
return ops
|
||||||
|
}
|
||||||
|
|
||||||
|
// FailExecution records an invocation that failed before issuing any SQL.
|
||||||
|
func FailExecution(opts *WriterOptions, dialect string, operations []ExecutionOperation, cause error) error {
|
||||||
|
report := &ExecutionReport{Dialect: dialect, TotalStatements: len(operations), SkippedStatements: len(operations), Outcome: "not_started", Operations: append([]ExecutionOperation(nil), operations...)}
|
||||||
|
report.StartTime = time.Now().Format(time.RFC3339Nano)
|
||||||
|
report.EndTime = report.StartTime
|
||||||
|
for i := range report.Operations {
|
||||||
|
report.Operations[i].StatementNumber = i + 1
|
||||||
|
report.Operations[i].Status = "skipped"
|
||||||
|
}
|
||||||
|
return FinishExecution(opts, report, cause)
|
||||||
|
}
|
||||||
@@ -0,0 +1,96 @@
|
|||||||
|
package writers
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestExecutionOutcomes(t *testing.T) {
|
||||||
|
for _, atomic := range []bool{false, true} {
|
||||||
|
for _, continuation := range []bool{false, true} {
|
||||||
|
t.Run(strings.Join([]string{map[bool]string{false: "independent", true: "atomic"}[atomic], map[bool]string{false: "stop", true: "continue"}[continuation]}, "/"), func(t *testing.T) {
|
||||||
|
attempted, rollbacks := 0, 0
|
||||||
|
cause := errors.New("injected failure")
|
||||||
|
backend := ExecutionBackend{
|
||||||
|
Exec: func(_ context.Context, i int, _ string) error {
|
||||||
|
attempted++
|
||||||
|
if i == 1 {
|
||||||
|
return cause
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
},
|
||||||
|
Begin: func(context.Context) error { return nil }, Commit: func(context.Context) error { t.Fatal("must not commit"); return nil },
|
||||||
|
Rollback: func(context.Context) error { rollbacks++; return nil },
|
||||||
|
}
|
||||||
|
opts := &WriterOptions{ContinueOnError: continuation, AtomicExecution: atomic}
|
||||||
|
report := &ExecutionReport{Dialect: "test"}
|
||||||
|
err := RunExecution(context.Background(), opts, report, StatementOperations([]string{"one", "bad", "three"}), backend)
|
||||||
|
if atomic && continuation {
|
||||||
|
if err == nil || attempted != 0 || report.SkippedStatements != 3 {
|
||||||
|
t.Fatalf("invalid options must not execute: %+v %v", report, err)
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
var failure *ExecutionFailure
|
||||||
|
if !errors.As(err, &failure) || !errors.Is(err, cause) {
|
||||||
|
t.Fatalf("must preserve typed report and cause: %v", err)
|
||||||
|
}
|
||||||
|
if report.FailedStatements != 1 || report.TotalStatements != 3 || report.Errors[0].StatementNumber != 2 {
|
||||||
|
t.Fatalf("failure report: %+v", report)
|
||||||
|
}
|
||||||
|
if atomic {
|
||||||
|
if rollbacks != 1 || report.ExecutedStatements != 0 || report.RolledBackStatements != 1 || report.Operations[0].Status != "rolled_back" || report.Outcome != "rolled_back" {
|
||||||
|
t.Fatalf("rollback report: %+v", report)
|
||||||
|
}
|
||||||
|
} else if continuation {
|
||||||
|
if attempted != 3 || report.ExecutedStatements != 2 || report.SkippedStatements != 0 {
|
||||||
|
t.Fatalf("continuation report: %+v", report)
|
||||||
|
}
|
||||||
|
} else if attempted != 2 || report.ExecutedStatements != 1 || report.SkippedStatements != 1 {
|
||||||
|
t.Fatalf("stop report: %+v", report)
|
||||||
|
}
|
||||||
|
if opts.Metadata["execution_report"] != report {
|
||||||
|
t.Fatal("typed results missing")
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestExecutionUncertainFinalization(t *testing.T) {
|
||||||
|
for _, failStatement := range []bool{false, true} {
|
||||||
|
t.Run(map[bool]string{false: "commit", true: "rollback"}[failStatement], func(t *testing.T) {
|
||||||
|
finalErr := errors.New("connection lost during finalization")
|
||||||
|
backend := ExecutionBackend{
|
||||||
|
Begin: func(context.Context) error { return nil },
|
||||||
|
Exec: func(_ context.Context, i int, _ string) error {
|
||||||
|
if failStatement && i == 1 {
|
||||||
|
return errors.New("statement failed")
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
},
|
||||||
|
Commit: func(context.Context) error { return finalErr }, Rollback: func(context.Context) error { return finalErr },
|
||||||
|
}
|
||||||
|
report := &ExecutionReport{}
|
||||||
|
err := RunExecution(context.Background(), &WriterOptions{AtomicExecution: true}, report, StatementOperations([]string{"one", "two"}), backend)
|
||||||
|
if !errors.Is(err, finalErr) || report.Outcome != "unknown" || report.Operations[0].Status != "unknown" || report.RolledBackStatements != 0 || report.ExecutedStatements != 0 {
|
||||||
|
t.Fatalf("uncertain outcome misreported: %+v %v", report, err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestExecutionReportFailureAndFatalPreflight(t *testing.T) {
|
||||||
|
opts := &WriterOptions{ContinueOnError: true, Metadata: map[string]interface{}{"report_path": filepath.Join(t.TempDir(), "missing", "report.json")}}
|
||||||
|
report := &ExecutionReport{}
|
||||||
|
calls := 0
|
||||||
|
ops := []ExecutionOperation{{Statement: "preflight", StopOnFailure: true}, {Statement: "must not run"}}
|
||||||
|
err := RunExecution(context.Background(), opts, report, ops, ExecutionBackend{Exec: func(context.Context, int, string) error { calls++; return errors.New("invalid data") }})
|
||||||
|
var failure *ExecutionFailure
|
||||||
|
if !errors.As(err, &failure) || !strings.Contains(err.Error(), "requested execution report") || calls != 1 || report.SkippedStatements != 1 {
|
||||||
|
t.Fatalf("must retain both failures and stop: %+v %v", report, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
+15
-15
@@ -502,30 +502,30 @@ func (w *Writer) executeDatabaseSQL(db *models.Database, connString string) erro
|
|||||||
// Connect to database
|
// Connect to database
|
||||||
dbConn, err := sql.Open("mssql", connString)
|
dbConn, err := sql.Open("mssql", connString)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("failed to connect to database: %w", err)
|
return writers.FailExecution(w.options, "SQL Server", writers.StatementOperations(statements), fmt.Errorf("failed to connect to database: %w", err))
|
||||||
}
|
}
|
||||||
defer dbConn.Close()
|
defer dbConn.Close()
|
||||||
|
|
||||||
ctx := context.Background()
|
ctx := context.Background()
|
||||||
if err = dbConn.PingContext(ctx); err != nil {
|
if err = dbConn.PingContext(ctx); err != nil {
|
||||||
return fmt.Errorf("failed to ping database: %w", err)
|
return writers.FailExecution(w.options, "SQL Server", writers.StatementOperations(statements), fmt.Errorf("failed to ping database: %w", err))
|
||||||
}
|
}
|
||||||
|
|
||||||
// Execute statements
|
var tx *sql.Tx
|
||||||
executedCount := 0
|
backend := writers.ExecutionBackend{
|
||||||
for i, stmt := range statements {
|
Exec: func(ctx context.Context, _ int, stmt string) error {
|
||||||
fmt.Fprintf(os.Stderr, "Executing statement %d/%d...\n", i+1, len(statements))
|
if tx != nil {
|
||||||
|
_, err := tx.ExecContext(ctx, stmt)
|
||||||
if _, execErr := dbConn.ExecContext(ctx, stmt); execErr != nil {
|
return err
|
||||||
fmt.Fprintf(os.Stderr, "⚠ Warning: Statement failed: %v\n", execErr)
|
|
||||||
continue
|
|
||||||
}
|
}
|
||||||
|
_, err := dbConn.ExecContext(ctx, stmt)
|
||||||
executedCount++
|
return err
|
||||||
|
},
|
||||||
|
Begin: func(ctx context.Context) error { var err error; tx, err = dbConn.BeginTx(ctx, nil); return err },
|
||||||
|
Commit: func(context.Context) error { return tx.Commit() },
|
||||||
|
Rollback: func(context.Context) error { return tx.Rollback() },
|
||||||
}
|
}
|
||||||
|
return writers.RunExecution(ctx, w.options, &writers.ExecutionReport{Dialect: "SQL Server"}, writers.StatementOperations(statements), backend)
|
||||||
fmt.Fprintf(os.Stderr, "✓ Successfully executed %d statements\n", executedCount)
|
|
||||||
return nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// generateStatements renders the same script WriteDatabase would produce and
|
// generateStatements renders the same script WriteDatabase would produce and
|
||||||
|
|||||||
@@ -0,0 +1,69 @@
|
|||||||
|
package mssql
|
||||||
|
|
||||||
|
import (
|
||||||
|
"database/sql"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"git.warky.dev/wdevs/relspecgo/pkg/models"
|
||||||
|
"git.warky.dev/wdevs/relspecgo/pkg/writers"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestLive_ExecutionFailureModes(t *testing.T) {
|
||||||
|
connection := os.Getenv("RELSPEC_TEST_MSSQL_CONN")
|
||||||
|
if connection == "" {
|
||||||
|
t.Skip("RELSPEC_TEST_MSSQL_CONN not set")
|
||||||
|
}
|
||||||
|
conn, err := sql.Open("mssql", connection)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer conn.Close()
|
||||||
|
for _, mode := range []string{"stop", "continue", "atomic"} {
|
||||||
|
t.Run(mode, func(t *testing.T) {
|
||||||
|
prefix := fmt.Sprintf("audit_%d_", time.Now().UnixNano())
|
||||||
|
db := models.InitDatabase("audit")
|
||||||
|
schema := models.InitSchema("dbo")
|
||||||
|
for _, name := range []string{"first", "bad", "last"} {
|
||||||
|
table := models.InitTable(prefix+name, "dbo")
|
||||||
|
col := models.InitColumn("id", table.Name, "dbo")
|
||||||
|
col.Type = "integer"
|
||||||
|
if name == "bad" {
|
||||||
|
duplicate := models.InitColumn("id", table.Name, "dbo")
|
||||||
|
duplicate.Type = "integer"
|
||||||
|
table.Columns["duplicate"] = duplicate
|
||||||
|
}
|
||||||
|
table.Columns[col.Name] = col
|
||||||
|
schema.Tables = append(schema.Tables, table)
|
||||||
|
}
|
||||||
|
db.Schemas = append(db.Schemas, schema)
|
||||||
|
t.Cleanup(func() {
|
||||||
|
for _, name := range []string{"first", "bad", "last"} {
|
||||||
|
_, _ = conn.Exec("DROP TABLE IF EXISTS dbo." + prefix + name)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
opts := &writers.WriterOptions{ContinueOnError: mode == "continue", AtomicExecution: mode == "atomic", Metadata: map[string]interface{}{"connection_string": connection}}
|
||||||
|
err := NewWriter(opts).WriteDatabase(db)
|
||||||
|
var failure *writers.ExecutionFailure
|
||||||
|
if !errors.As(err, &failure) || failure.Report.FailedStatements == 0 {
|
||||||
|
t.Fatalf("expected structured error: %v", err)
|
||||||
|
}
|
||||||
|
for _, name := range []string{"first", "last"} {
|
||||||
|
var count int
|
||||||
|
if err := conn.QueryRow("SELECT count(*) FROM sys.tables WHERE name=?", prefix+name).Scan(&count); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
want := mode != "atomic" && (name == "first" || mode == "continue")
|
||||||
|
if (count == 1) != want {
|
||||||
|
t.Fatalf("%s exists=%v want=%v", name, count == 1, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if mode == "atomic" && (failure.Report.Outcome != "rolled_back" || failure.Report.RolledBackStatements == 0 || failure.Report.ExecutedStatements != 0) {
|
||||||
|
t.Fatalf("incorrect rollback report: %+v", failure.Report)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,145 @@
|
|||||||
|
package mysql
|
||||||
|
|
||||||
|
import (
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"git.warky.dev/wdevs/relspecgo/pkg/models"
|
||||||
|
"git.warky.dev/wdevs/relspecgo/pkg/transform"
|
||||||
|
"git.warky.dev/wdevs/relspecgo/pkg/writers"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestStrictConversionPreflightsAllObjects(t *testing.T) {
|
||||||
|
db := shopDB()
|
||||||
|
unsupported := models.InitTable("unsupported", "shop")
|
||||||
|
col := models.InitColumn("opaque", "unsupported", "shop")
|
||||||
|
col.Type = "unknown_type"
|
||||||
|
unsupported.Columns[col.Name] = col
|
||||||
|
index := models.InitIndex("partial", "unsupported", "shop")
|
||||||
|
index.Columns = []string{col.Name}
|
||||||
|
index.Where = "opaque IS NOT NULL"
|
||||||
|
index.Include = []string{"other"}
|
||||||
|
unsupported.Indexes[index.Name] = index
|
||||||
|
db.Schemas[0].Tables = append(db.Schemas[0].Tables, unsupported)
|
||||||
|
out := filepath.Join(t.TempDir(), "existing.sql")
|
||||||
|
if err := os.WriteFile(out, []byte("keep existing contents"), 0o600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
opts := &writers.WriterOptions{StrictConversion: true, OutputPath: out}
|
||||||
|
err := NewWriter(opts).WriteDatabase(db)
|
||||||
|
for _, want := range []string{"lossy MySQL conversion rejected", "unknown type", "partial predicate", "INCLUDE"} {
|
||||||
|
if err == nil || !strings.Contains(err.Error(), want) {
|
||||||
|
t.Fatalf("expected %s: %v", want, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
bytes, err := os.ReadFile(out)
|
||||||
|
if err != nil || string(bytes) != "keep existing contents" {
|
||||||
|
t.Fatalf("destination changed: %q %v", bytes, err)
|
||||||
|
}
|
||||||
|
opts.Metadata = map[string]any{"connection_string": "invalid DSN"}
|
||||||
|
if err := NewWriter(opts).WriteDatabase(db); err == nil || !strings.Contains(err.Error(), "lossy MySQL") {
|
||||||
|
t.Fatalf("connected before strict validation: %v", err)
|
||||||
|
}
|
||||||
|
opts.StrictConversion = false
|
||||||
|
opts.Metadata = nil
|
||||||
|
var warnings []string
|
||||||
|
opts.Warning = func(s string) { warnings = append(warnings, s) }
|
||||||
|
if err := NewWriter(opts).WriteDatabase(db); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(warnings) != 3 {
|
||||||
|
t.Fatalf("warnings: %v", warnings)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestUnsupportedModelFeaturesDiagnosed(t *testing.T) {
|
||||||
|
db := shopDB()
|
||||||
|
s := db.Schemas[0]
|
||||||
|
table := s.Tables[0]
|
||||||
|
s.Metadata["mysql_unsupported"] = []any{"trigger shop.audit", "partitioned table shop.archive"}
|
||||||
|
c := table.Columns["id"]
|
||||||
|
c.Type = "timestamptz"
|
||||||
|
c.Identity = true
|
||||||
|
c.Metadata["mysql_srs_id"] = 4326
|
||||||
|
check := models.InitConstraint("deferred", models.CheckConstraint)
|
||||||
|
check.Expression = "id > 0"
|
||||||
|
yes := true
|
||||||
|
check.Deferrable = &yes
|
||||||
|
table.Constraints[check.Name] = check
|
||||||
|
i := models.InitIndex("gin", "users", "shop")
|
||||||
|
i.Type = "gin"
|
||||||
|
i.Columns = []string{"id"}
|
||||||
|
table.Indexes[i.Name] = i
|
||||||
|
_, err := NewWriter(&writers.WriterOptions{StrictConversion: true}).Statements(db)
|
||||||
|
for _, want := range []string{"trigger shop.audit", "partitioned table", "time zone", "identity", "mysql_srs_id", "deferrable", "unsupported index type gin"} {
|
||||||
|
if err == nil || !strings.Contains(err.Error(), want) {
|
||||||
|
t.Errorf("missing diagnostic %q: %v", want, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCanonicalDimensionsAndNativeEdit(t *testing.T) {
|
||||||
|
db := shopDB()
|
||||||
|
c := db.Schemas[0].Tables[0].Columns["age"]
|
||||||
|
c.Type = "decimal"
|
||||||
|
c.Precision = 12
|
||||||
|
c.Scale = 4
|
||||||
|
if got := columnType(c); got != "DECIMAL(12,4)" {
|
||||||
|
t.Fatal(got)
|
||||||
|
}
|
||||||
|
c.Type = "uint64"
|
||||||
|
c.Precision = 0
|
||||||
|
c.Scale = 0
|
||||||
|
if got := columnType(c); got != "BIGINT UNSIGNED" {
|
||||||
|
t.Fatal(got)
|
||||||
|
}
|
||||||
|
c.Metadata["mysql_type"] = "bigint unsigned"
|
||||||
|
c.Metadata["mysql_type_signature"] = signature(c)
|
||||||
|
c.Type = "int64"
|
||||||
|
_, err := NewWriter(&writers.WriterOptions{StrictConversion: true}).Statements(db)
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "override native declaration") {
|
||||||
|
t.Fatalf("native metadata silently overrode edit: %v", err)
|
||||||
|
}
|
||||||
|
if got := columnType(c); got != "BIGINT" {
|
||||||
|
t.Fatal(got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestLowercaseIndexPartsPreserveSQL(t *testing.T) {
|
||||||
|
db := shopDB()
|
||||||
|
table := db.Schemas[0].Tables[0]
|
||||||
|
c := models.InitColumn("Mixed", "users", "shop")
|
||||||
|
c.Type = "string"
|
||||||
|
table.Columns[c.Name] = c
|
||||||
|
i := models.InitIndex("MixedIndex", "users", "shop")
|
||||||
|
i.KeyParts = []models.IndexPart{{Name: "Mixed", Length: 8, Descending: true}, {Expression: "lower(`Mixed`)"}}
|
||||||
|
table.Indexes[i.Name] = i
|
||||||
|
model, err := transform.LowercaseIdentifiers(db)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
parts := model.Schemas[0].Tables[0].Indexes["mixedindex"].KeyParts
|
||||||
|
if parts[0].Name != "mixed" || parts[1].Expression != "lower(`Mixed`)" || i.KeyParts[0].Name != "Mixed" {
|
||||||
|
t.Fatalf("lowercase parts: %+v", parts)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestLowercasePrimaryIndexIsNotDuplicated(t *testing.T) {
|
||||||
|
db := shopDB()
|
||||||
|
table := db.Schemas[0].Tables[0]
|
||||||
|
i := models.InitIndex("PRIMARY", table.Name, table.Schema)
|
||||||
|
i.Unique = true
|
||||||
|
i.Type = "btree"
|
||||||
|
i.Columns = []string{"id"}
|
||||||
|
i.KeyParts = []models.IndexPart{{Name: "id", Descending: true}}
|
||||||
|
table.Indexes[i.Name] = i
|
||||||
|
statements, err := NewWriter(&writers.WriterOptions{StrictConversion: true, LowercaseIdentifiers: true}).Statements(db)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if !strings.Contains(statements[0], "PRIMARY KEY (`id` DESC)") || strings.Contains(statements[0], "KEY `primary`") {
|
||||||
|
t.Fatalf("primary key duplicated or attributes lost: %s", statements[0])
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,184 @@
|
|||||||
|
package mysql
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"sort"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"git.warky.dev/wdevs/relspecgo/pkg/mariadb"
|
||||||
|
"git.warky.dev/wdevs/relspecgo/pkg/models"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Diagnostics cover modeled schema semantics and features discovered by the
|
||||||
|
// MySQL reader. They do not certify arbitrary SQL expressions or catalog grants.
|
||||||
|
func conversionDiagnostics(db *models.Database) []string {
|
||||||
|
var out []string
|
||||||
|
warn := func(path, feature string) { out = append(out, path+": "+feature) }
|
||||||
|
unsupported := func(m map[string]any) {
|
||||||
|
switch values := m["mysql_unsupported"].(type) {
|
||||||
|
case []string:
|
||||||
|
for _, v := range values {
|
||||||
|
warn(v, "unsupported object")
|
||||||
|
}
|
||||||
|
case []any:
|
||||||
|
for _, v := range values {
|
||||||
|
warn(fmt.Sprint(v), "unsupported object")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
metadata := func(path string, m map[string]any, allowed string) {
|
||||||
|
if len(models.GetDirectives(m)) > 0 {
|
||||||
|
warn(path, "dialect directives are not emitted")
|
||||||
|
}
|
||||||
|
for _, key := range keys(m) {
|
||||||
|
if strings.HasPrefix(key, "mysql_") && !strings.Contains(" "+allowed+" ", " "+key+" ") {
|
||||||
|
warn(path, "unsupported metadata "+key+": "+fmt.Sprint(m[key]))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
metadata("database", db.Metadata, "mysql_unsupported mysql_sql_mode")
|
||||||
|
modeWarning := func(path string, m map[string]any) {
|
||||||
|
if mode := unsupportedMode(metaString(m, "mysql_sql_mode")); mode != "" {
|
||||||
|
warn(path, "unsupported source SQL mode "+mode+"; generated SQL requires standard quotes and backslash escaping")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
modeWarning("database", db.Metadata)
|
||||||
|
unsupported(db.Metadata)
|
||||||
|
if len(db.Domains) > 0 {
|
||||||
|
warn("database", "domains are not emitted")
|
||||||
|
}
|
||||||
|
for _, s := range db.Schemas {
|
||||||
|
path := s.Name
|
||||||
|
metadata(path, s.Metadata, "mysql_unsupported mysql_sql_mode")
|
||||||
|
modeWarning(path, s.Metadata)
|
||||||
|
unsupported(s.Metadata)
|
||||||
|
if len(s.Views) > 0 {
|
||||||
|
warn(path, "views are not emitted")
|
||||||
|
}
|
||||||
|
if len(s.Sequences) > 0 {
|
||||||
|
warn(path, "standalone sequences are not emitted")
|
||||||
|
}
|
||||||
|
if len(s.Enums) > 0 {
|
||||||
|
warn(path, "standalone enum types are not emitted; use native column enum types")
|
||||||
|
}
|
||||||
|
if len(s.Scripts) > 0 {
|
||||||
|
warn(path, "scripts are not emitted")
|
||||||
|
}
|
||||||
|
if s.Owner != "" || len(s.Permissions) > 0 {
|
||||||
|
warn(path, "ownership and permissions are not emitted")
|
||||||
|
}
|
||||||
|
for _, t := range s.Tables {
|
||||||
|
tp := path + "." + t.Name
|
||||||
|
modeWarning(tp, t.Metadata)
|
||||||
|
if t.Tablespace != "" {
|
||||||
|
warn(tp, "tablespace is not emitted")
|
||||||
|
}
|
||||||
|
metadata(tp, t.Metadata, "mysql_engine mysql_collation mysql_create_options mysql_auto_increment mysql_sql_mode")
|
||||||
|
if options := metaString(t.Metadata, "mysql_create_options"); options != "" {
|
||||||
|
warn(tp, "table CREATE_OPTIONS are not emitted: "+options)
|
||||||
|
}
|
||||||
|
for _, name := range keys(t.Columns) {
|
||||||
|
c := t.Columns[name]
|
||||||
|
cp := tp + "." + c.Name
|
||||||
|
metadata(cp, c.Metadata, "mysql_type mysql_type_signature mysql_charset mysql_datetime_precision mysql_default_expression mysql_default_sql mysql_default_value mysql_generation_storage mysql_invisible mysql_on_update")
|
||||||
|
if _, known := mariadb.CanonicalToMariaDBTypes[strings.ToLower(c.Type)]; !known && metaString(c.Metadata, "mysql_type") == "" {
|
||||||
|
warn(cp, "unknown type "+c.Type+" maps to "+columnType(c))
|
||||||
|
}
|
||||||
|
if native := metaString(c.Metadata, "mysql_type"); native != "" && metaString(c.Metadata, "mysql_type_signature") != signature(c) {
|
||||||
|
warn(cp, "edited canonical type/dimensions override native declaration "+native)
|
||||||
|
}
|
||||||
|
if c.Type == "timestamptz" {
|
||||||
|
warn(cp, "time zone semantics are not supported")
|
||||||
|
}
|
||||||
|
if c.Type == "jsonb" {
|
||||||
|
warn(cp, "jsonb storage semantics become JSON")
|
||||||
|
}
|
||||||
|
if c.Identity {
|
||||||
|
warn(cp, "identity semantics are not emitted; use auto_increment")
|
||||||
|
}
|
||||||
|
if c.Generated && c.GenerationExpression == "" {
|
||||||
|
warn(cp, "generated column has no expression")
|
||||||
|
}
|
||||||
|
if !c.Generated && c.GenerationExpression != "" {
|
||||||
|
warn(cp, "generation expression requires generated=true")
|
||||||
|
}
|
||||||
|
if c.Generated && (c.Default != nil || c.AutoIncrement) {
|
||||||
|
warn(cp, "generated column cannot preserve a default or auto_increment")
|
||||||
|
}
|
||||||
|
if storage := metaString(c.Metadata, "mysql_generation_storage"); storage != "" && storage != "STORED" && storage != "VIRTUAL" {
|
||||||
|
warn(cp, "unsupported generation storage "+storage)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, name := range keys(t.Constraints) {
|
||||||
|
c := t.Constraints[name]
|
||||||
|
cp := tp + " constraint " + c.Name
|
||||||
|
metadata(cp, c.Metadata, "mysql_enforced")
|
||||||
|
if c.Deferrable != nil && *c.Deferrable || c.InitiallyDeferred != nil && *c.InitiallyDeferred {
|
||||||
|
warn(cp, "deferrable constraints are not supported")
|
||||||
|
}
|
||||||
|
switch c.Type {
|
||||||
|
case models.PrimaryKeyConstraint, models.UniqueConstraint:
|
||||||
|
case models.CheckConstraint:
|
||||||
|
if c.Expression == "" {
|
||||||
|
warn(cp, "check constraint has no expression")
|
||||||
|
}
|
||||||
|
case models.ForeignKeyConstraint:
|
||||||
|
engine := metaString(t.Metadata, "mysql_engine")
|
||||||
|
if engine != "" && !strings.EqualFold(engine, "InnoDB") && !strings.EqualFold(engine, "NDB") {
|
||||||
|
warn(cp, "foreign keys are not supported by engine "+engine)
|
||||||
|
}
|
||||||
|
for _, action := range []string{c.OnDelete, c.OnUpdate} {
|
||||||
|
switch strings.ToUpper(action) {
|
||||||
|
case "", "CASCADE", "SET NULL", "RESTRICT", "NO ACTION":
|
||||||
|
default:
|
||||||
|
warn(cp, "unsupported referential action is not emitted: "+action)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
default:
|
||||||
|
warn(cp, "unsupported constraint type "+string(c.Type))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, name := range keys(t.Indexes) {
|
||||||
|
i := t.Indexes[name]
|
||||||
|
ip := tp + " index " + i.Name
|
||||||
|
metadata(ip, i.Metadata, "mysql_visible")
|
||||||
|
if i.Where != "" {
|
||||||
|
warn(ip, "partial predicate is not emitted")
|
||||||
|
}
|
||||||
|
if len(i.Include) > 0 {
|
||||||
|
warn(ip, "INCLUDE columns are not emitted")
|
||||||
|
}
|
||||||
|
if i.Concurrent {
|
||||||
|
warn(ip, "concurrent creation is not supported")
|
||||||
|
}
|
||||||
|
if strings.EqualFold(i.Type, "hash") && !strings.EqualFold(metaString(t.Metadata, "mysql_engine"), "MEMORY") {
|
||||||
|
warn(ip, "HASH index requires a compatible engine; InnoDB converts it to BTREE")
|
||||||
|
}
|
||||||
|
switch strings.ToUpper(i.Type) {
|
||||||
|
case "", "BTREE", "HASH", "FULLTEXT", "SPATIAL":
|
||||||
|
default:
|
||||||
|
warn(ip, "unsupported index type "+i.Type+"; index is not emitted")
|
||||||
|
}
|
||||||
|
for _, p := range i.KeyParts {
|
||||||
|
if (p.Name == "") == (p.Expression == "") {
|
||||||
|
warn(ip, "key part requires exactly one name or expression")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, name := range keys(t.Relationships) {
|
||||||
|
r := t.Relationships[name]
|
||||||
|
if t.Constraints[r.ForeignKey] == nil {
|
||||||
|
warn(tp+" relationship "+r.Name, "no backing foreign key constraint; relationship is not emitted")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
sort.Strings(out)
|
||||||
|
unique := out[:0]
|
||||||
|
for _, d := range out {
|
||||||
|
if len(unique) == 0 || unique[len(unique)-1] != d {
|
||||||
|
unique = append(unique, d)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return unique
|
||||||
|
}
|
||||||
@@ -0,0 +1,142 @@
|
|||||||
|
package mysql_test
|
||||||
|
|
||||||
|
import (
|
||||||
|
"database/sql"
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
driver "github.com/go-sql-driver/mysql"
|
||||||
|
|
||||||
|
"git.warky.dev/wdevs/relspecgo/pkg/models"
|
||||||
|
|
||||||
|
"git.warky.dev/wdevs/relspecgo/pkg/readers"
|
||||||
|
rmysql "git.warky.dev/wdevs/relspecgo/pkg/readers/mysql"
|
||||||
|
"git.warky.dev/wdevs/relspecgo/pkg/writers"
|
||||||
|
wmysql "git.warky.dev/wdevs/relspecgo/pkg/writers/mysql"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestLiveUnsupportedObjectsDiagnosed(t *testing.T) {
|
||||||
|
conn := os.Getenv("RELSPEC_TEST_MYSQL_CONN")
|
||||||
|
if conn == "" {
|
||||||
|
t.Skip("RELSPEC_TEST_MYSQL_CONN is not set")
|
||||||
|
}
|
||||||
|
db, err := sql.Open("mysql", conn)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
t.Cleanup(func() { db.Close() })
|
||||||
|
prefix := fmt.Sprintf("unsupported_%d", time.Now().UnixNano())
|
||||||
|
table, view, trigger := prefix+"_table", prefix+"_view", prefix+"_trigger"
|
||||||
|
t.Cleanup(func() {
|
||||||
|
for _, stmt := range []string{"DROP VIEW IF EXISTS " + view, "DROP TRIGGER IF EXISTS " + trigger, "DROP TABLE IF EXISTS " + table} {
|
||||||
|
if _, err := db.Exec(stmt); err != nil {
|
||||||
|
t.Error(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
for _, stmt := range []string{
|
||||||
|
"CREATE TABLE " + table + " (id INT PRIMARY KEY) PARTITION BY HASH(id) PARTITIONS 2",
|
||||||
|
"CREATE VIEW " + view + " AS SELECT id FROM " + table,
|
||||||
|
"CREATE TRIGGER " + trigger + " BEFORE INSERT ON " + table + " FOR EACH ROW SET NEW.id = NEW.id + 1",
|
||||||
|
} {
|
||||||
|
if _, err := db.Exec(stmt); err != nil {
|
||||||
|
t.Fatalf("%s: %v", stmt, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
model, err := rmysql.NewReader(&readers.ReaderOptions{ConnectionString: conn}).ReadDatabase()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
path := filepath.Join(t.TempDir(), "schema.sql")
|
||||||
|
if err := os.WriteFile(path, []byte("original"), 0o600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
err = wmysql.NewWriter(&writers.WriterOptions{StrictConversion: true, OutputPath: path}).WriteDatabase(model)
|
||||||
|
for _, want := range []string{"partitioned table", view, trigger} {
|
||||||
|
if err == nil || !strings.Contains(err.Error(), want) {
|
||||||
|
t.Errorf("missing diagnostic %s: %v", want, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
bytes, readErr := os.ReadFile(path)
|
||||||
|
if readErr != nil || string(bytes) != "original" {
|
||||||
|
t.Fatalf("strict failure touched file: %s %v", bytes, readErr)
|
||||||
|
}
|
||||||
|
// Schema-filtered/library writes retain discovery diagnostics as well.
|
||||||
|
err = wmysql.NewWriter(&writers.WriterOptions{StrictConversion: true}).WriteSchema(model.Schemas[0])
|
||||||
|
if err == nil || !strings.Contains(err.Error(), trigger) {
|
||||||
|
t.Fatalf("filtered schema lost diagnostics: %v", err)
|
||||||
|
}
|
||||||
|
var warnings []string
|
||||||
|
_, err = wmysql.NewWriter(&writers.WriterOptions{Warning: func(s string) { warnings = append(warnings, s) }}).Statements(model)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
for _, want := range []string{view, trigger, table} {
|
||||||
|
found := false
|
||||||
|
for _, warning := range warnings {
|
||||||
|
if strings.Contains(warning, want) {
|
||||||
|
found = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !found {
|
||||||
|
t.Errorf("no warning for %s: %v", want, warnings)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestLiveUnsupportedSQLModeFailsBeforeDDL(t *testing.T) {
|
||||||
|
conn := os.Getenv("RELSPEC_TEST_MYSQL_CONN")
|
||||||
|
if conn == "" {
|
||||||
|
t.Skip("RELSPEC_TEST_MYSQL_CONN is not set")
|
||||||
|
}
|
||||||
|
for _, mode := range []string{"NO_BACKSLASH_ESCAPES", "ANSI_QUOTES"} {
|
||||||
|
t.Run(mode, func(t *testing.T) {
|
||||||
|
cfg, err := driver.ParseDSN(conn)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if cfg.Params == nil {
|
||||||
|
cfg.Params = map[string]string{}
|
||||||
|
}
|
||||||
|
cfg.Params["sql_mode"] = "'" + mode + "'"
|
||||||
|
modeConn := cfg.FormatDSN()
|
||||||
|
model, err := rmysql.NewReader(&readers.ReaderOptions{ConnectionString: modeConn}).ReadDatabase()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, err := wmysql.NewWriter(&writers.WriterOptions{StrictConversion: true}).Statements(model); err == nil || !strings.Contains(err.Error(), mode) {
|
||||||
|
t.Fatalf("source mode not diagnosed: %v", err)
|
||||||
|
}
|
||||||
|
db, err := sql.Open("mysql", conn)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer db.Close()
|
||||||
|
name := fmt.Sprintf("mode_%d", time.Now().UnixNano())
|
||||||
|
defer db.Exec("DROP TABLE IF EXISTS " + name)
|
||||||
|
target := models.InitDatabase(cfg.DBName)
|
||||||
|
s := models.InitSchema(cfg.DBName)
|
||||||
|
target.Schemas = append(target.Schemas, s)
|
||||||
|
table := models.InitTable(name, s.Name)
|
||||||
|
s.Tables = append(s.Tables, table)
|
||||||
|
c := models.InitColumn("id", name, s.Name)
|
||||||
|
c.Type = "int"
|
||||||
|
table.Columns[c.Name] = c
|
||||||
|
err = wmysql.NewWriter(&writers.WriterOptions{StrictConversion: true, Metadata: map[string]any{"connection_string": modeConn}}).WriteDatabase(target)
|
||||||
|
if err == nil || !strings.Contains(err.Error(), mode) {
|
||||||
|
t.Fatalf("target mode not rejected: %v", err)
|
||||||
|
}
|
||||||
|
var count int
|
||||||
|
if err := db.QueryRow("SELECT COUNT(*) FROM information_schema.TABLES WHERE TABLE_SCHEMA=? AND TABLE_NAME=?", cfg.DBName, name).Scan(&count); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if count != 0 {
|
||||||
|
t.Fatal("mode rejection occurred after DDL")
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,244 @@
|
|||||||
|
package mysql_test
|
||||||
|
|
||||||
|
import (
|
||||||
|
"database/sql"
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"reflect"
|
||||||
|
"sort"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"gopkg.in/yaml.v3"
|
||||||
|
|
||||||
|
driver "github.com/go-sql-driver/mysql"
|
||||||
|
|
||||||
|
"git.warky.dev/wdevs/relspecgo/pkg/models"
|
||||||
|
"git.warky.dev/wdevs/relspecgo/pkg/readers"
|
||||||
|
rmysql "git.warky.dev/wdevs/relspecgo/pkg/readers/mysql"
|
||||||
|
"git.warky.dev/wdevs/relspecgo/pkg/writers"
|
||||||
|
wmysql "git.warky.dev/wdevs/relspecgo/pkg/writers/mysql"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestLiveSchemaRoundTrip(t *testing.T) {
|
||||||
|
conn := os.Getenv("RELSPEC_TEST_MYSQL_CONN")
|
||||||
|
if conn == "" {
|
||||||
|
t.Skip("RELSPEC_TEST_MYSQL_CONN is not set")
|
||||||
|
}
|
||||||
|
db, err := sql.Open("mysql", conn)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
t.Cleanup(func() { db.Close() })
|
||||||
|
prefix := fmt.Sprintf("rt_%d", time.Now().UnixNano())
|
||||||
|
parent, child := prefix+"_zparent", prefix+"_achild"
|
||||||
|
q := func(n string) string { return "`" + strings.ReplaceAll(n, "`", "``") + "`" }
|
||||||
|
exec := func(stmt string) {
|
||||||
|
t.Helper()
|
||||||
|
if _, err := db.Exec(stmt); err != nil {
|
||||||
|
t.Fatalf("%v\n%s", err, stmt)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
t.Cleanup(func() {
|
||||||
|
for _, name := range []string{child + "_copy", child, parent + "_copy", parent} {
|
||||||
|
if _, err := db.Exec("DROP TABLE IF EXISTS " + q(name)); err != nil {
|
||||||
|
t.Errorf("cleanup %s: %v", name, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
exec("CREATE TABLE " + q(parent) + ` (
|
||||||
|
id BIGINT UNSIGNED NOT NULL AUTO_INCREMENT,
|
||||||
|
code CHAR(3) NOT NULL,
|
||||||
|
email VARCHAR(100) NOT NULL,
|
||||||
|
label VARCHAR(64) DEFAULT 'don''t (change)' COMMENT 'column''s comment',
|
||||||
|
amount DECIMAL(12,4) NOT NULL DEFAULT 3.2500,
|
||||||
|
medium MEDIUMINT UNSIGNED,
|
||||||
|
flags BIT(8) DEFAULT b'00001010',
|
||||||
|
status ENUM('one','two','don''t, split') DEFAULT 'don''t, split',
|
||||||
|
choices SET('a','b') DEFAULT 'a,b',
|
||||||
|
stamp TIMESTAMP(6) NULL DEFAULT CURRENT_TIMESTAMP(6) ON UPDATE CURRENT_TIMESTAMP(6),
|
||||||
|
duration TIME(4),
|
||||||
|
bytes VARBINARY(16),
|
||||||
|
binary_def VARBINARY(8) DEFAULT X'00275CFF',
|
||||||
|
slash VARCHAR(64) DEFAULT 'C:\\temp\\file',
|
||||||
|
uuid_value CHAR(36) DEFAULT (uuid()),
|
||||||
|
hidden_col INT INVISIBLE,
|
||||||
|
dt DATETIME(6) DEFAULT '2020-01-02 03:04:05.123456',
|
||||||
|
payload MEDIUMBLOB,
|
||||||
|
doc TEXT DEFAULT ('first;
|
||||||
|
-- data, not a comment
|
||||||
|
last'),
|
||||||
|
generated_amount DECIMAL(13,4) GENERATED ALWAYS AS (amount * 2) STORED,
|
||||||
|
virtual_label VARCHAR(80) GENERATED ALWAYS AS (concat('prefix:',lower(label))) VIRTUAL,
|
||||||
|
PRIMARY KEY (id,code),
|
||||||
|
UNIQUE KEY email_unique (email),
|
||||||
|
UNIQUE KEY pair_unique (label,email),
|
||||||
|
UNIQUE KEY unique_expression ((concat(lower(code),'-x')),email),
|
||||||
|
KEY prefix_desc (label(10) DESC),
|
||||||
|
KEY expression_key ((lower(email))),
|
||||||
|
KEY invisible_key (medium) INVISIBLE COMMENT 'index''s comment',
|
||||||
|
FULLTEXT KEY text_search (doc),
|
||||||
|
CONSTRAINT ` + q(prefix+"_amount_check") + ` CHECK (amount >= 0),
|
||||||
|
CONSTRAINT ` + q(prefix+"_optional_check") + ` CHECK (medium > 0 AND status <> 'forbidden;-- value') NOT ENFORCED
|
||||||
|
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci COMMENT='table''s comment'`)
|
||||||
|
exec("CREATE TABLE " + q(child) + ` (
|
||||||
|
id INT PRIMARY KEY,
|
||||||
|
parent_id BIGINT UNSIGNED,
|
||||||
|
parent_code CHAR(3),
|
||||||
|
parent_email VARCHAR(100),
|
||||||
|
CONSTRAINT ` + q(prefix+"_composite_fk") + ` FOREIGN KEY (parent_id,parent_code) REFERENCES ` + q(parent) + ` (id,code) ON DELETE SET NULL ON UPDATE CASCADE,
|
||||||
|
CONSTRAINT ` + q(prefix+"_email_fk") + ` FOREIGN KEY (parent_email) REFERENCES ` + q(parent) + ` (email) ON DELETE CASCADE ON UPDATE RESTRICT
|
||||||
|
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci`)
|
||||||
|
readerConfig, err := driver.ParseDSN(conn)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if readerConfig.Params == nil {
|
||||||
|
readerConfig.Params = map[string]string{}
|
||||||
|
}
|
||||||
|
readerConfig.Params["sql_quote_show_create"] = "0" // reader must ensure quoted native metadata
|
||||||
|
read := func() *models.Database {
|
||||||
|
t.Helper()
|
||||||
|
model, err := rmysql.NewReader(&readers.ReaderOptions{ConnectionString: readerConfig.FormatDSN()}).ReadDatabase()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
return model
|
||||||
|
}
|
||||||
|
model := read()
|
||||||
|
var selected []*models.Table
|
||||||
|
for _, table := range model.Schemas[0].Tables {
|
||||||
|
if table.Name == parent || table.Name == child {
|
||||||
|
selected = append(selected, table)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(selected) != 2 {
|
||||||
|
t.Fatalf("read %d fixture tables", len(selected))
|
||||||
|
}
|
||||||
|
model.Schemas[0].Tables = selected
|
||||||
|
model.Metadata = nil
|
||||||
|
model.Schemas[0].Metadata = nil // unrelated pre-existing test objects are outside this fixture
|
||||||
|
for _, table := range selected {
|
||||||
|
if table.Name == parent {
|
||||||
|
if table.Columns["id"].Type != "uint64" {
|
||||||
|
t.Fatal("unsigned type lost")
|
||||||
|
}
|
||||||
|
if table.Columns["amount"].Precision != 12 || table.Columns["amount"].Scale != 4 {
|
||||||
|
t.Fatal("decimal dimensions lost")
|
||||||
|
}
|
||||||
|
if table.Indexes["expression_key"].KeyParts[0].Expression == "" {
|
||||||
|
t.Fatal("functional index expression lost")
|
||||||
|
}
|
||||||
|
} else if len(table.Relationships) != 2 {
|
||||||
|
t.Fatalf("foreign key relationships collapsed: %d", len(table.Relationships))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// Exercise the serialized model boundary as well as the native reader objects.
|
||||||
|
encoded, err := json.Marshal(model)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
var target models.Database
|
||||||
|
if err := json.Unmarshal(encoded, &target); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
yamlModel, err := yaml.Marshal(&target)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
target = models.Database{}
|
||||||
|
if err := yaml.Unmarshal(yamlModel, &target); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
for _, table := range target.Schemas[0].Tables {
|
||||||
|
table.Name += "_copy"
|
||||||
|
for _, col := range table.Columns {
|
||||||
|
col.Table = table.Name
|
||||||
|
}
|
||||||
|
for _, index := range table.Indexes {
|
||||||
|
index.Table = table.Name
|
||||||
|
}
|
||||||
|
constraints := map[string]*models.Constraint{}
|
||||||
|
for _, c := range table.Constraints {
|
||||||
|
c.Table = table.Name
|
||||||
|
if c.Type == models.ForeignKeyConstraint {
|
||||||
|
c.ReferencedTable += "_copy"
|
||||||
|
}
|
||||||
|
if c.Type == models.ForeignKeyConstraint || c.Type == models.CheckConstraint {
|
||||||
|
c.Name += "_copy"
|
||||||
|
}
|
||||||
|
constraints[c.Name] = c
|
||||||
|
}
|
||||||
|
table.Constraints = constraints
|
||||||
|
for _, r := range table.Relationships {
|
||||||
|
r.FromTable += "_copy"
|
||||||
|
r.ToTable += "_copy"
|
||||||
|
r.ForeignKey += "_copy"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
opts := &writers.WriterOptions{StrictConversion: true, Metadata: map[string]any{"connection_string": conn}}
|
||||||
|
if err := wmysql.NewWriter(opts).WriteDatabase(&target); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
show := func(table string) string {
|
||||||
|
t.Helper()
|
||||||
|
var name, ddl string
|
||||||
|
if err := db.QueryRow("SHOW CREATE TABLE "+q(table)).Scan(&name, &ddl); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
ddl = strings.ReplaceAll(ddl, "_copy", "")
|
||||||
|
// Index creation order is not semantic. Keep column and constraint order intact.
|
||||||
|
lines := strings.Split(ddl, "\n")
|
||||||
|
var indexes []string
|
||||||
|
for _, line := range lines {
|
||||||
|
if strings.HasPrefix(strings.TrimSpace(line), "KEY ") {
|
||||||
|
indexes = append(indexes, line)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
sort.Strings(indexes)
|
||||||
|
n := 0
|
||||||
|
for i, line := range lines {
|
||||||
|
if strings.HasPrefix(strings.TrimSpace(line), "KEY ") {
|
||||||
|
lines[i] = indexes[n]
|
||||||
|
n++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return strings.Join(lines, "\n")
|
||||||
|
}
|
||||||
|
for _, name := range []string{parent, child} {
|
||||||
|
a, b := show(name), show(name+"_copy")
|
||||||
|
if a != b {
|
||||||
|
t.Errorf("schema mismatch\nsource:\n%s\ntarget:\n%s", a, b)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
again := read()
|
||||||
|
for _, table := range again.Schemas[0].Tables {
|
||||||
|
if table.Name == parent+"_copy" && !reflect.DeepEqual(table.Columns["amount"].Metadata, selected[1].Columns["amount"].Metadata) {
|
||||||
|
t.Error("type metadata changed on second read")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// Verify constraints and defaults as behavior, after comparing untouched schemas.
|
||||||
|
exec("INSERT INTO " + q(parent+"_copy") + " (id,code,email) VALUES (18446744073709551614,'abc','one@example.com')")
|
||||||
|
exec("INSERT INTO " + q(child+"_copy") + " (id,parent_id,parent_code,parent_email) VALUES (1,18446744073709551614,'abc','one@example.com')")
|
||||||
|
if _, err := db.Exec("INSERT INTO " + q(child+"_copy") + " (id,parent_id,parent_code) VALUES (2,123,'abc')"); err == nil {
|
||||||
|
t.Error("foreign key is not enforced")
|
||||||
|
}
|
||||||
|
if _, err := db.Exec("UPDATE " + q(parent+"_copy") + " SET amount=-1"); err == nil {
|
||||||
|
t.Error("check is not enforced")
|
||||||
|
}
|
||||||
|
var label, doc, binaryHex, slash string
|
||||||
|
var flags int
|
||||||
|
if err := db.QueryRow("SELECT label,doc,flags+0,HEX(binary_def),slash FROM "+q(parent+"_copy")).Scan(&label, &doc, &flags, &binaryHex, &slash); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
for _, table := range read().Schemas[0].Tables {
|
||||||
|
if table.Name == parent+"_copy" && table.Metadata["mysql_auto_increment"] != "18446744073709551615" {
|
||||||
|
t.Errorf("large auto-increment counter lost: %v", table.Metadata["mysql_auto_increment"])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if label != "don't (change)" || doc != "first;\n-- data, not a comment\nlast" || flags != 10 || binaryHex != "00275CFF" || slash != `C:\temp\file` {
|
||||||
|
t.Errorf("defaults corrupted: %q %q %d %s %q", label, doc, flags, binaryHex, slash)
|
||||||
|
}
|
||||||
|
}
|
||||||
+352
-107
@@ -7,12 +7,14 @@ import (
|
|||||||
"io"
|
"io"
|
||||||
"os"
|
"os"
|
||||||
"sort"
|
"sort"
|
||||||
|
"strconv"
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
_ "github.com/go-sql-driver/mysql"
|
_ "github.com/go-sql-driver/mysql"
|
||||||
|
|
||||||
"git.warky.dev/wdevs/relspecgo/pkg/mariadb"
|
"git.warky.dev/wdevs/relspecgo/pkg/mariadb"
|
||||||
"git.warky.dev/wdevs/relspecgo/pkg/models"
|
"git.warky.dev/wdevs/relspecgo/pkg/models"
|
||||||
|
"git.warky.dev/wdevs/relspecgo/pkg/transform"
|
||||||
"git.warky.dev/wdevs/relspecgo/pkg/writers"
|
"git.warky.dev/wdevs/relspecgo/pkg/writers"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -22,83 +24,202 @@ type Writer struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func NewWriter(options *writers.WriterOptions) *Writer { return &Writer{options: options} }
|
func NewWriter(options *writers.WriterOptions) *Writer { return &Writer{options: options} }
|
||||||
func (w *Writer) WriteDatabase(db *models.Database) error {
|
|
||||||
|
// Statements preflights the whole model, emits conversion diagnostics, and returns
|
||||||
|
// executable units. No output files or database connections are opened here.
|
||||||
|
func (w *Writer) Statements(db *models.Database) ([]string, error) {
|
||||||
if w.options == nil {
|
if w.options == nil {
|
||||||
return fmt.Errorf("writer options are required")
|
return nil, fmt.Errorf("writer options are required")
|
||||||
}
|
}
|
||||||
if conn, ok := w.options.Metadata["connection_string"].(string); ok && conn != "" {
|
if db == nil {
|
||||||
return w.execute(db, conn)
|
return nil, fmt.Errorf("database is required")
|
||||||
}
|
}
|
||||||
release, err := w.openOutput()
|
if w.options.LowercaseIdentifiers {
|
||||||
if err != nil {
|
var err error
|
||||||
return err
|
db, err = transform.LowercaseIdentifiers(db)
|
||||||
}
|
|
||||||
defer release()
|
|
||||||
return w.writeDatabaseDDL(db)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (w *Writer) writeDatabaseDDL(db *models.Database) error {
|
|
||||||
if _, err := fmt.Fprintf(w.writer, "-- MySQL Database Schema\n-- Database: %s\n-- Generated by RelSpec\n\n", db.Name); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
for _, s := range db.Schemas {
|
|
||||||
if err := w.WriteSchema(s); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// openOutput points w.writer at the configured destination (output file or
|
|
||||||
// stdout) when none is set, and returns a func that releases it again.
|
|
||||||
func (w *Writer) openOutput() (func(), error) {
|
|
||||||
if w.writer != nil {
|
|
||||||
return func() {}, nil
|
|
||||||
}
|
|
||||||
if w.options != nil && w.options.OutputPath != "" {
|
|
||||||
f, err := os.Create(w.options.OutputPath)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
w.writer = f
|
|
||||||
return func() {
|
|
||||||
f.Close()
|
|
||||||
w.writer = nil
|
|
||||||
}, nil
|
|
||||||
}
|
}
|
||||||
w.writer = os.Stdout
|
diagnostics := conversionDiagnostics(db)
|
||||||
return func() { w.writer = nil }, nil
|
if len(diagnostics) > 0 && w.options.StrictConversion {
|
||||||
|
return nil, fmt.Errorf("lossy MySQL conversion rejected:\n- %s", strings.Join(diagnostics, "\n- "))
|
||||||
|
}
|
||||||
|
for _, d := range diagnostics {
|
||||||
|
if w.options.Warning != nil {
|
||||||
|
w.options.Warning(d)
|
||||||
|
} else {
|
||||||
|
fmt.Fprintln(os.Stderr, "warning: MySQL conversion: "+d)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return databaseStatements(db), nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (w *Writer) WriteSchema(s *models.Schema) error {
|
func (w *Writer) WriteDatabase(db *models.Database) error {
|
||||||
release, err := w.openOutput()
|
statements, err := w.Statements(db)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
defer release()
|
if conn, ok := w.options.Metadata["connection_string"].(string); ok && conn != "" {
|
||||||
for _, t := range s.Tables {
|
return w.execute(statements, conn)
|
||||||
if err := w.writeTable(s, t); err != nil {
|
}
|
||||||
|
if w.writer != nil {
|
||||||
|
return writeStatements(w.writer, statements, db.Name)
|
||||||
|
}
|
||||||
|
if w.options.OutputPath == "" {
|
||||||
|
return writeStatements(os.Stdout, statements, db.Name)
|
||||||
|
}
|
||||||
|
f, err := os.Create(w.options.OutputPath)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
err = writeStatements(f, statements, db.Name)
|
||||||
|
closeErr := f.Close()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return closeErr
|
||||||
|
}
|
||||||
|
|
||||||
|
func writeStatements(out io.Writer, statements []string, name string) error {
|
||||||
|
name = strings.NewReplacer("\n", " ", "\r", " ").Replace(name)
|
||||||
|
if _, err := fmt.Fprintf(out, "-- Database: %s\n", name); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if _, err := io.WriteString(out, "-- MySQL table schema; generated by RelSpec\n\n"); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
for _, stmt := range statements {
|
||||||
|
if _, err := io.WriteString(out, stmt); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (w *Writer) WriteTable(t *models.Table) error {
|
func (w *Writer) WriteSchema(s *models.Schema) error {
|
||||||
release, err := w.openOutput()
|
return w.WriteDatabase(&models.Database{Schemas: []*models.Schema{s}})
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
defer release()
|
|
||||||
return w.writeTable(nil, t)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func (w *Writer) writeTable(s *models.Schema, t *models.Table) error {
|
func (w *Writer) WriteTable(t *models.Table) error {
|
||||||
name := t.Name
|
return w.WriteSchema(&models.Schema{Name: "", Tables: []*models.Table{t}})
|
||||||
|
}
|
||||||
|
|
||||||
|
func qualified(schema, name string) string {
|
||||||
|
if schema == "" {
|
||||||
|
return quote(name)
|
||||||
|
}
|
||||||
|
return quote(schema) + "." + quote(name)
|
||||||
|
}
|
||||||
|
|
||||||
|
func keys[T any](m map[string]T) []string {
|
||||||
|
out := make([]string, 0, len(m))
|
||||||
|
for k := range m {
|
||||||
|
out = append(out, k)
|
||||||
|
}
|
||||||
|
sort.Strings(out)
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
func metaString(m map[string]any, key string) string { v, _ := m[key].(string); return v }
|
||||||
|
func metaBool(m map[string]any, key string, def bool) bool {
|
||||||
|
v, ok := m[key].(bool)
|
||||||
|
if ok {
|
||||||
|
return v
|
||||||
|
}
|
||||||
|
return def
|
||||||
|
}
|
||||||
|
|
||||||
|
func signature(c *models.Column) string {
|
||||||
|
return fmt.Sprintf("%s/%d/%d/%d", c.Type, c.Length, c.Precision, c.Scale)
|
||||||
|
}
|
||||||
|
|
||||||
|
func columnType(c *models.Column) string {
|
||||||
|
if native := metaString(c.Metadata, "mysql_type"); native != "" && metaString(c.Metadata, "mysql_type_signature") == signature(c) {
|
||||||
|
return native
|
||||||
|
}
|
||||||
|
typ := mariadb.ConvertCanonicalToMariaDB(c.Type)
|
||||||
|
switch strings.ToLower(c.Type) {
|
||||||
|
case "string":
|
||||||
|
if c.Length > 0 {
|
||||||
|
typ = fmt.Sprintf("VARCHAR(%d)", c.Length)
|
||||||
|
}
|
||||||
|
case "decimal":
|
||||||
|
if c.Precision > 0 {
|
||||||
|
typ = fmt.Sprintf("DECIMAL(%d,%d)", c.Precision, c.Scale)
|
||||||
|
}
|
||||||
|
case "timestamp", "time":
|
||||||
|
if c.Precision > 0 {
|
||||||
|
typ += fmt.Sprintf("(%d)", c.Precision)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return typ
|
||||||
|
}
|
||||||
|
|
||||||
|
// MySQL string literals use doubled quotes and escaped backslashes. The output
|
||||||
|
// assumes the server's standard SQL mode (NO_BACKSLASH_ESCAPES is not enabled).
|
||||||
|
func literal(s string) string {
|
||||||
|
return "'" + strings.ReplaceAll(strings.ReplaceAll(s, "\\", "\\\\"), "'", "''") + "'"
|
||||||
|
}
|
||||||
|
|
||||||
|
func defaultSQL(c *models.Column) string {
|
||||||
|
value := fmt.Sprint(c.Default)
|
||||||
|
if native := metaString(c.Metadata, "mysql_default_sql"); native != "" && metaString(c.Metadata, "mysql_default_value") == value {
|
||||||
|
return native
|
||||||
|
}
|
||||||
|
if strings.HasPrefix(metaString(c.Metadata, "mysql_type"), "bit(") && strings.HasPrefix(value, "b'") && strings.HasSuffix(value, "'") && strings.Trim(value[2:len(value)-1], "01") == "" {
|
||||||
|
return value
|
||||||
|
}
|
||||||
|
if metaBool(c.Metadata, "mysql_default_expression", false) {
|
||||||
|
if strings.HasPrefix(strings.ToUpper(value), "CURRENT_TIMESTAMP") {
|
||||||
|
return value
|
||||||
|
}
|
||||||
|
return "(" + value + ")"
|
||||||
|
}
|
||||||
|
switch strings.ToLower(c.Type) {
|
||||||
|
case "text", "json", "jsonb", "bytea":
|
||||||
|
return "(" + literal(value) + ")"
|
||||||
|
case "string", "uuid", "date", "time", "timestamp", "timestamptz":
|
||||||
|
return literal(value)
|
||||||
|
case "bool", "boolean":
|
||||||
|
if value == "true" {
|
||||||
|
return "1"
|
||||||
|
}
|
||||||
|
if value == "false" {
|
||||||
|
return "0"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// Numeric defaults are values, never executable fragments.
|
||||||
|
if _, err := strconv.ParseFloat(value, 64); err == nil {
|
||||||
|
return value
|
||||||
|
}
|
||||||
|
return literal(value)
|
||||||
|
}
|
||||||
|
|
||||||
|
func indexParts(i *models.Index) string {
|
||||||
|
if len(i.KeyParts) == 0 {
|
||||||
|
return quoted(i.Columns)
|
||||||
|
}
|
||||||
|
parts := make([]string, 0, len(i.KeyParts))
|
||||||
|
for _, p := range i.KeyParts {
|
||||||
|
s := quote(p.Name)
|
||||||
|
if p.Expression != "" {
|
||||||
|
s = "(" + p.Expression + ")"
|
||||||
|
}
|
||||||
|
if p.Length > 0 {
|
||||||
|
s += fmt.Sprintf("(%d)", p.Length)
|
||||||
|
}
|
||||||
|
if p.Descending {
|
||||||
|
s += " DESC"
|
||||||
|
}
|
||||||
|
parts = append(parts, s)
|
||||||
|
}
|
||||||
|
return strings.Join(parts, ", ")
|
||||||
|
}
|
||||||
|
|
||||||
|
func tableStatement(s *models.Schema, t *models.Table) string {
|
||||||
|
schema := ""
|
||||||
if s != nil {
|
if s != nil {
|
||||||
name = fmt.Sprintf("%s.%s", quote(s.Name), quote(t.Name))
|
schema = s.Name
|
||||||
} else {
|
|
||||||
name = quote(name)
|
|
||||||
}
|
}
|
||||||
cols := make([]*models.Column, 0, len(t.Columns))
|
cols := make([]*models.Column, 0, len(t.Columns))
|
||||||
for _, c := range t.Columns {
|
for _, c := range t.Columns {
|
||||||
@@ -113,92 +234,198 @@ func (w *Writer) writeTable(s *models.Schema, t *models.Table) error {
|
|||||||
defs := []string{}
|
defs := []string{}
|
||||||
pk := []string{}
|
pk := []string{}
|
||||||
for _, c := range cols {
|
for _, c := range cols {
|
||||||
d := fmt.Sprintf(" %s %s", quote(c.Name), mariadb.ConvertCanonicalToMariaDB(c.Type))
|
d := " " + quote(c.Name) + " " + columnType(c)
|
||||||
if c.Length > 0 && strings.EqualFold(c.Type, "string") {
|
if charset := metaString(c.Metadata, "mysql_charset"); charset != "" && c.Collation != metaString(t.Metadata, "mysql_collation") {
|
||||||
d = fmt.Sprintf(" %s VARCHAR(%d)", quote(c.Name), c.Length)
|
d += " CHARACTER SET " + quote(charset)
|
||||||
|
}
|
||||||
|
if c.Collation != "" && c.Collation != metaString(t.Metadata, "mysql_collation") {
|
||||||
|
d += " COLLATE " + quote(c.Collation)
|
||||||
|
}
|
||||||
|
if c.Generated {
|
||||||
|
d += " GENERATED ALWAYS AS (" + c.GenerationExpression + ")"
|
||||||
|
if strings.EqualFold(metaString(c.Metadata, "mysql_generation_storage"), "STORED") {
|
||||||
|
d += " STORED"
|
||||||
|
} else {
|
||||||
|
d += " VIRTUAL"
|
||||||
|
}
|
||||||
}
|
}
|
||||||
if c.NotNull {
|
if c.NotNull {
|
||||||
d += " NOT NULL"
|
d += " NOT NULL"
|
||||||
|
} else if !c.Generated {
|
||||||
|
d += " NULL"
|
||||||
}
|
}
|
||||||
if c.AutoIncrement {
|
if c.AutoIncrement {
|
||||||
d += " AUTO_INCREMENT"
|
d += " AUTO_INCREMENT"
|
||||||
}
|
}
|
||||||
if c.Default != nil {
|
if c.Default != nil && !c.Generated {
|
||||||
d += fmt.Sprintf(" DEFAULT %s", writers.QuoteDefaultValue(fmt.Sprint(c.Default), c.Type))
|
d += " DEFAULT " + defaultSQL(c)
|
||||||
|
}
|
||||||
|
if upd := metaString(c.Metadata, "mysql_on_update"); upd != "" {
|
||||||
|
d += " ON UPDATE " + upd
|
||||||
|
}
|
||||||
|
if metaBool(c.Metadata, "mysql_invisible", false) {
|
||||||
|
d += " INVISIBLE"
|
||||||
|
}
|
||||||
|
if c.Comment != "" {
|
||||||
|
d += " COMMENT " + literal(c.Comment)
|
||||||
}
|
}
|
||||||
defs = append(defs, d)
|
defs = append(defs, d)
|
||||||
if c.IsPrimaryKey {
|
if c.IsPrimaryKey {
|
||||||
pk = append(pk, quote(c.Name))
|
pk = append(pk, c.Name)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
constraintNames := make([]string, 0, len(t.Constraints))
|
for _, name := range keys(t.Constraints) {
|
||||||
for name := range t.Constraints {
|
|
||||||
constraintNames = append(constraintNames, name)
|
|
||||||
}
|
|
||||||
sort.Strings(constraintNames)
|
|
||||||
for _, name := range constraintNames {
|
|
||||||
c := t.Constraints[name]
|
c := t.Constraints[name]
|
||||||
if c.Type == models.PrimaryKeyConstraint {
|
if c.Type == models.PrimaryKeyConstraint {
|
||||||
pk = nil
|
pk = c.Columns
|
||||||
for _, n := range c.Columns {
|
|
||||||
pk = append(pk, quote(n))
|
|
||||||
}
|
|
||||||
break
|
break
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if len(pk) > 0 {
|
var primaryIndex *models.Index
|
||||||
defs = append(defs, " PRIMARY KEY ("+strings.Join(pk, ", ")+")")
|
for name, i := range t.Indexes {
|
||||||
|
if strings.EqualFold(name, "PRIMARY") {
|
||||||
|
primaryIndex = i
|
||||||
|
break
|
||||||
}
|
}
|
||||||
for _, name := range constraintNames {
|
}
|
||||||
|
if i := primaryIndex; i != nil {
|
||||||
|
defs = append(defs, " PRIMARY KEY ("+indexParts(i)+")")
|
||||||
|
} else if len(pk) > 0 {
|
||||||
|
defs = append(defs, " PRIMARY KEY ("+quoted(pk)+")")
|
||||||
|
}
|
||||||
|
for _, name := range keys(t.Constraints) {
|
||||||
c := t.Constraints[name]
|
c := t.Constraints[name]
|
||||||
if c.Type == models.UniqueConstraint {
|
switch c.Type {
|
||||||
defs = append(defs, fmt.Sprintf(" CONSTRAINT %s UNIQUE (%s)", quote(c.Name), quoted(c.Columns)))
|
case models.UniqueConstraint:
|
||||||
|
if t.Indexes[c.Name] == nil {
|
||||||
|
defs = append(defs, " CONSTRAINT "+quote(c.Name)+" UNIQUE ("+quoted(c.Columns)+")")
|
||||||
|
}
|
||||||
|
case models.CheckConstraint:
|
||||||
|
d := " CONSTRAINT " + quote(c.Name) + " CHECK (" + c.Expression + ")"
|
||||||
|
if !metaBool(c.Metadata, "mysql_enforced", true) {
|
||||||
|
d += " NOT ENFORCED"
|
||||||
|
}
|
||||||
|
defs = append(defs, d)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
sql := fmt.Sprintf("CREATE TABLE IF NOT EXISTS %s (\n%s\n) ENGINE=InnoDB;\n\n", name, strings.Join(defs, ",\n"))
|
for _, name := range keys(t.Indexes) {
|
||||||
if _, err := io.WriteString(w.writer, sql); err != nil {
|
i := t.Indexes[name]
|
||||||
return err
|
if strings.EqualFold(name, "PRIMARY") {
|
||||||
|
continue
|
||||||
}
|
}
|
||||||
return nil
|
method := strings.ToUpper(i.Type)
|
||||||
|
if method != "" && method != "BTREE" && method != "HASH" && method != "FULLTEXT" && method != "SPATIAL" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
kind := "KEY"
|
||||||
|
if i.Unique {
|
||||||
|
kind = "UNIQUE KEY"
|
||||||
|
}
|
||||||
|
if method == "FULLTEXT" || method == "SPATIAL" {
|
||||||
|
kind = method + " KEY"
|
||||||
|
}
|
||||||
|
d := " " + kind + " " + quote(i.Name) + " (" + indexParts(i) + ")"
|
||||||
|
if method == "HASH" {
|
||||||
|
d += " USING " + method
|
||||||
|
}
|
||||||
|
if !metaBool(i.Metadata, "mysql_visible", true) {
|
||||||
|
d += " INVISIBLE"
|
||||||
|
}
|
||||||
|
if i.Comment != "" {
|
||||||
|
d += " COMMENT " + literal(i.Comment)
|
||||||
|
}
|
||||||
|
defs = append(defs, d)
|
||||||
|
}
|
||||||
|
engine := metaString(t.Metadata, "mysql_engine")
|
||||||
|
if engine == "" {
|
||||||
|
engine = "InnoDB"
|
||||||
|
}
|
||||||
|
tail := " ENGINE=" + quote(engine)
|
||||||
|
if engine == "InnoDB" {
|
||||||
|
tail = " ENGINE=InnoDB"
|
||||||
|
}
|
||||||
|
if coll := metaString(t.Metadata, "mysql_collation"); coll != "" {
|
||||||
|
tail += " COLLATE=" + quote(coll)
|
||||||
|
}
|
||||||
|
if auto := metaString(t.Metadata, "mysql_auto_increment"); auto != "" {
|
||||||
|
if _, err := strconv.ParseUint(auto, 10, 64); err == nil {
|
||||||
|
tail += " AUTO_INCREMENT=" + auto
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if t.Comment != "" {
|
||||||
|
tail += " COMMENT=" + literal(t.Comment)
|
||||||
|
}
|
||||||
|
return "CREATE TABLE IF NOT EXISTS " + qualified(schema, t.Name) + " (\n" + strings.Join(defs, ",\n") + "\n)" + tail + ";\n\n"
|
||||||
}
|
}
|
||||||
|
|
||||||
func (w *Writer) execute(dbm *models.Database, conn string) error {
|
func databaseStatements(db *models.Database) []string {
|
||||||
var b strings.Builder
|
var statements []string
|
||||||
old := w.writer
|
for _, s := range db.Schemas {
|
||||||
w.writer = &b
|
for _, t := range s.Tables {
|
||||||
if err := w.writeDatabaseDDL(dbm); err != nil {
|
statements = append(statements, tableStatement(s, t))
|
||||||
w.writer = old
|
}
|
||||||
return err
|
}
|
||||||
|
// References are added after all tables, supporting cycles and forward references.
|
||||||
|
for _, s := range db.Schemas {
|
||||||
|
for _, t := range s.Tables {
|
||||||
|
for _, name := range keys(t.Constraints) {
|
||||||
|
c := t.Constraints[name]
|
||||||
|
if c.Type != models.ForeignKeyConstraint {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
rs := c.ReferencedSchema
|
||||||
|
if rs == "" {
|
||||||
|
rs = s.Name
|
||||||
|
}
|
||||||
|
stmt := "ALTER TABLE " + qualified(s.Name, t.Name) + " ADD CONSTRAINT " + quote(c.Name) + " FOREIGN KEY (" + quoted(c.Columns) + ") REFERENCES " + qualified(rs, c.ReferencedTable) + " (" + quoted(c.ReferencedColumns) + ")"
|
||||||
|
if validAction(c.OnDelete) && c.OnDelete != "" {
|
||||||
|
stmt += " ON DELETE " + strings.ToUpper(c.OnDelete)
|
||||||
|
}
|
||||||
|
if validAction(c.OnUpdate) && c.OnUpdate != "" {
|
||||||
|
stmt += " ON UPDATE " + strings.ToUpper(c.OnUpdate)
|
||||||
|
}
|
||||||
|
statements = append(statements, stmt+";\n\n")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return statements
|
||||||
|
}
|
||||||
|
|
||||||
|
func (w *Writer) execute(statements []string, conn string) error {
|
||||||
|
if w.options.AtomicExecution {
|
||||||
|
return fmt.Errorf("MySQL DDL implicitly commits; atomic execution is unsupported")
|
||||||
}
|
}
|
||||||
w.writer = old
|
|
||||||
db, err := sql.Open("mysql", conn)
|
db, err := sql.Open("mysql", conn)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("failed to connect: %w", err)
|
return fmt.Errorf("failed to connect: %w", err)
|
||||||
}
|
}
|
||||||
defer db.Close()
|
defer db.Close()
|
||||||
for _, stmt := range strings.Split(b.String(), ";\n") {
|
session, err := db.Conn(context.Background())
|
||||||
stmt = stripComments(strings.TrimSpace(stmt))
|
if err != nil {
|
||||||
if stmt == "" {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
if _, err := db.ExecContext(context.Background(), stmt); err != nil {
|
|
||||||
return fmt.Errorf("failed to execute SQL: %w", err)
|
return fmt.Errorf("failed to execute SQL: %w", err)
|
||||||
}
|
}
|
||||||
|
defer session.Close()
|
||||||
|
var mode string
|
||||||
|
if err := session.QueryRowContext(context.Background(), "SELECT @@SESSION.sql_mode").Scan(&mode); err != nil {
|
||||||
|
return fmt.Errorf("read MySQL SQL mode: %w", err)
|
||||||
|
}
|
||||||
|
if mode = unsupportedMode(mode); mode != "" {
|
||||||
|
return fmt.Errorf("MySQL execution requires standard quotes and backslash escaping; %s is enabled", mode)
|
||||||
|
}
|
||||||
|
var failures []string
|
||||||
|
for n, stmt := range statements {
|
||||||
|
if _, err := session.ExecContext(context.Background(), stmt); err != nil {
|
||||||
|
failures = append(failures, fmt.Sprintf("statement %d: %v", n+1, err))
|
||||||
|
if !w.options.ContinueOnError {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(failures) > 0 {
|
||||||
|
return fmt.Errorf("failed to execute SQL: %s", strings.Join(failures, "; "))
|
||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func stripComments(sqlText string) string {
|
|
||||||
lines := strings.Split(sqlText, "\n")
|
|
||||||
kept := lines[:0]
|
|
||||||
for _, line := range lines {
|
|
||||||
if !strings.HasPrefix(strings.TrimSpace(line), "--") {
|
|
||||||
kept = append(kept, line)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return strings.TrimSpace(strings.Join(kept, "\n"))
|
|
||||||
}
|
|
||||||
|
|
||||||
func quote(s string) string { return "`" + strings.ReplaceAll(s, "`", "``") + "`" }
|
func quote(s string) string { return "`" + strings.ReplaceAll(s, "`", "``") + "`" }
|
||||||
func quoted(xs []string) string {
|
func quoted(xs []string) string {
|
||||||
out := make([]string, len(xs))
|
out := make([]string, len(xs))
|
||||||
@@ -207,3 +434,21 @@ func quoted(xs []string) string {
|
|||||||
}
|
}
|
||||||
return strings.Join(out, ", ")
|
return strings.Join(out, ", ")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func validAction(action string) bool {
|
||||||
|
switch strings.ToUpper(action) {
|
||||||
|
case "", "CASCADE", "SET NULL", "RESTRICT", "NO ACTION":
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
func unsupportedMode(mode string) string {
|
||||||
|
var modes []string
|
||||||
|
for _, unsupported := range []string{"NO_BACKSLASH_ESCAPES", "ANSI_QUOTES"} {
|
||||||
|
if strings.Contains(strings.ToUpper(mode), unsupported) {
|
||||||
|
modes = append(modes, unsupported)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return strings.Join(modes, ", ")
|
||||||
|
}
|
||||||
|
|||||||
@@ -9,13 +9,6 @@ import (
|
|||||||
"git.warky.dev/wdevs/relspecgo/pkg/writers"
|
"git.warky.dev/wdevs/relspecgo/pkg/writers"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestStripComments(t *testing.T) {
|
|
||||||
got := stripComments("-- header\nCREATE TABLE `users` (\n `id` INT\n);")
|
|
||||||
if strings.HasPrefix(got, "--") || !strings.HasPrefix(got, "CREATE TABLE") {
|
|
||||||
t.Fatalf("stripComments() = %q", got)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestWriterGeneratesMySQLDDL(t *testing.T) {
|
func TestWriterGeneratesMySQLDDL(t *testing.T) {
|
||||||
db := models.InitDatabase("app")
|
db := models.InitDatabase("app")
|
||||||
s := models.InitSchema("app")
|
s := models.InitSchema("app")
|
||||||
@@ -42,3 +35,26 @@ func TestWriterGeneratesMySQLDDL(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestDatabaseStatementsKeepEmbeddedDelimiters(t *testing.T) {
|
||||||
|
db := models.InitDatabase("app")
|
||||||
|
s := models.InitSchema("app")
|
||||||
|
db.Schemas = append(db.Schemas, s)
|
||||||
|
for _, name := range []string{"one", "two"} {
|
||||||
|
table := models.InitTable(name, s.Name)
|
||||||
|
col := models.InitColumn("text", name, s.Name)
|
||||||
|
col.Type = "text"
|
||||||
|
col.Default = "first;\n-- data, not a comment\nlast"
|
||||||
|
table.Columns[col.Name] = col
|
||||||
|
s.Tables = append(s.Tables, table)
|
||||||
|
}
|
||||||
|
statements := databaseStatements(db)
|
||||||
|
if len(statements) != 2 {
|
||||||
|
t.Fatalf("got %d statements", len(statements))
|
||||||
|
}
|
||||||
|
for _, stmt := range statements {
|
||||||
|
if !strings.Contains(stmt, "first;\n-- data, not a comment\nlast") {
|
||||||
|
t.Fatalf("default corrupted: %s", stmt)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -17,6 +17,28 @@ The PostgreSQL Writer converts RelSpec's internal database model representation
|
|||||||
- Includes audit triggers (optional)
|
- Includes audit triggers (optional)
|
||||||
- Handles PostgreSQL-specific data types
|
- Handles PostgreSQL-specific data types
|
||||||
|
|
||||||
|
## Identifier and value handling
|
||||||
|
|
||||||
|
Generated identifiers are always double-quoted, including reserved words, and
|
||||||
|
retain the model's case by default. Supply raw model names, without SQL quoting.
|
||||||
|
Identifiers containing double quotes are escaped by doubling them; NUL is rejected
|
||||||
|
before database/file output. Catalog checks, comments, audit values, and regclass
|
||||||
|
names use SQL literal quoting; generated procedural/function bodies choose dollar
|
||||||
|
delimiters absent from their rendered content. Names in generated line comments
|
||||||
|
cannot create new SQL lines.
|
||||||
|
|
||||||
|
Use CLI `--lowercase`, job option `lowercase_identifiers: true`, or PostgreSQL
|
||||||
|
`WriterOptions.LowercaseIdentifiers: true` to lowercase model names and structured
|
||||||
|
references. The transformation copies the model and rejects name collisions. Other
|
||||||
|
library writers can use `transform.LowercaseIdentifiers` before writing.
|
||||||
|
|
||||||
|
SQL expressions are executable input: types, defaults, generated expressions,
|
||||||
|
checks, index expressions/predicates, dialect directive clauses, audit user
|
||||||
|
expressions, and `pgsql_using` / `pgsql_preflight` are not sanitized or case-folded.
|
||||||
|
Adjust identifier references inside these expressions yourself when lowercasing.
|
||||||
|
Enum values, comments, and other data retain their case. Migration matching respects
|
||||||
|
identifier case; `"Users"` and `"users"` are distinct objects.
|
||||||
|
|
||||||
## Usage
|
## Usage
|
||||||
|
|
||||||
### Basic Example
|
### Basic Example
|
||||||
@@ -346,3 +368,67 @@ metadata:
|
|||||||
- Default values are properly quoted
|
- Default values are properly quoted
|
||||||
- Constraint names follow PostgreSQL conventions
|
- Constraint names follow PostgreSQL conventions
|
||||||
- Compatible with PostgreSQL 12+
|
- Compatible with PostgreSQL 12+
|
||||||
|
|
||||||
|
## Live execution failures
|
||||||
|
|
||||||
|
Live PostgreSQL output stops after the first failed statement by default and returns
|
||||||
|
`*writers.ExecutionFailure` (also available as `*pgsql.ExecutionFailure`; inspect it
|
||||||
|
with `errors.As`). `WriterOptions.ContinueOnError` or job
|
||||||
|
`options.continue_on_error: true` attempts later statements but still returns an error
|
||||||
|
if any fail, preventing dependent jobs from running. A failed type preflight always
|
||||||
|
stops live execution, including when continuation is enabled.
|
||||||
|
|
||||||
|
PostgreSQL, SQLite, SQL Server, and the PostgreSQL script runner share execution
|
||||||
|
reports through `WriterOptions.Metadata["execution_report"]`. Optional
|
||||||
|
`Metadata["report_path"]` writes the report as JSON; write and close failures are
|
||||||
|
errors. Reports include each operation's status and original number, plus applied,
|
||||||
|
failed, skipped, and rolled-back counts. Blank and comment entries count as skipped.
|
||||||
|
Unknown commit/rollback outcomes are explicitly marked `unknown` and are never
|
||||||
|
reported as confirmed rollback. Connection failures record all planned operations as
|
||||||
|
skipped. Raw scripts have different reporting limits; see the sqlexec README.
|
||||||
|
|
||||||
|
Generated DDL uses independent statements by default: earlier changes can remain
|
||||||
|
committed after failure. Set `WriterOptions.AtomicExecution = true` to use one
|
||||||
|
transaction with PostgreSQL, SQLite, or SQL Server generated DDL. For database output
|
||||||
|
jobs, use:
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
options:
|
||||||
|
atomic_execution: true
|
||||||
|
```
|
||||||
|
|
||||||
|
Atomic execution and continuation are mutually exclusive. Rollback covers database
|
||||||
|
transactional effects; sequence advancement and external effects in SQL expressions
|
||||||
|
are outside that guarantee. Concurrency locks, migration history, and resumable
|
||||||
|
migration management remain separate work (audit A09).
|
||||||
|
|
||||||
|
## Column type conversions
|
||||||
|
|
||||||
|
Type changes propagate errors and never implicitly rename or replace a column.
|
||||||
|
Before schema changes, full-DDL and diff plans evaluate every existing value through
|
||||||
|
the conversion expression and target cast. Diff preflights run before dropping
|
||||||
|
constraints/indexes or creating sequences; database-wide preflights run before
|
||||||
|
changing any schema. Matching types, missing tables/columns, generated columns, and
|
||||||
|
identity columns need no conversion check.
|
||||||
|
|
||||||
|
Column metadata in JSON/YAML models supports two optional SQL expressions:
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
metadata:
|
||||||
|
pgsql_using: "split_part(name, ' ', 1)::integer"
|
||||||
|
pgsql_preflight: "name ~ '^[0-9]+ units$'"
|
||||||
|
```
|
||||||
|
|
||||||
|
`pgsql_using` replaces the default `column::target_type` expression in `USING`.
|
||||||
|
`pgsql_preflight` is a row predicate; false or NULL rejects the conversion. These
|
||||||
|
fields must contain nonempty strings. Both are trusted executable SQL, like defaults
|
||||||
|
and check expressions; they are not values or a sandbox. Preflights evaluate the
|
||||||
|
conversion expression, so use expressions without side effects. New types/extensions
|
||||||
|
needed by an existing-column conversion must already be available for preflight.
|
||||||
|
|
||||||
|
Preflight checks can race concurrent writes and do not prove that locks, defaults,
|
||||||
|
or every dependency will permit the ALTER. The ALTER still validates the conversion;
|
||||||
|
use atomic execution when preceding operations also need rollback. Generated SQL
|
||||||
|
clients must stop on preflight failure; live RelSpec execution enforces this even
|
||||||
|
with continuation enabled. Rename/replacement/backfill operations still require a
|
||||||
|
separately reviewed migration.
|
||||||
|
|||||||
@@ -22,8 +22,9 @@ func TestCurrentColumnHasDescription(t *testing.T) {
|
|||||||
want bool
|
want bool
|
||||||
}{
|
}{
|
||||||
{"nil table", nil, &models.Column{Name: "email", Description: "x"}, false},
|
{"nil table", nil, &models.Column{Name: "email", Description: "x"}, false},
|
||||||
{"match ignoring case and whitespace", table, &models.Column{Name: "email", Description: "the email"}, true},
|
{"match ignoring whitespace", table, &models.Column{Name: "Email", Description: "the email"}, true},
|
||||||
{"different description", table, &models.Column{Name: "email", Description: "other"}, false},
|
{"different description", table, &models.Column{Name: "Email", Description: "other"}, false},
|
||||||
|
{"different identifier case", table, &models.Column{Name: "email", Description: "the email"}, false},
|
||||||
{"column missing", table, &models.Column{Name: "age", Description: "x"}, false},
|
{"column missing", table, &models.Column{Name: "age", Description: "x"}, false},
|
||||||
}
|
}
|
||||||
for _, tt := range tests {
|
for _, tt := range tests {
|
||||||
@@ -41,12 +42,12 @@ func TestExecuteCommentColumn(t *testing.T) {
|
|||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
got, err := te.ExecuteCommentColumn(CommentColumnData{
|
got, err := te.ExecuteCommentColumn(CommentColumnData{
|
||||||
SchemaName: "public", TableName: "users", ColumnName: "email", Comment: "it''s",
|
SchemaName: "public", TableName: "users", ColumnName: "email", Comment: "it's",
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
if !strings.Contains(got, "COMMENT ON COLUMN") || !strings.Contains(got, "public.users") ||
|
if !strings.Contains(got, "COMMENT ON COLUMN") || !strings.Contains(got, "\"public\".\"users\"") ||
|
||||||
!strings.Contains(got, "email") || !strings.Contains(got, "IS 'it''s';") {
|
!strings.Contains(got, "email") || !strings.Contains(got, "IS 'it''s';") {
|
||||||
t.Errorf("unexpected output: %s", got)
|
t.Errorf("unexpected output: %s", got)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -57,8 +57,8 @@ func TestDiffStatements(t *testing.T) {
|
|||||||
current: func() *models.Database {
|
current: func() *models.Database {
|
||||||
return diffTestDB("public", map[string]map[string]string{"users": {"id": "integer"}})
|
return diffTestDB("public", map[string]map[string]string{"users": {"id": "integer"}})
|
||||||
},
|
},
|
||||||
wantContain: []string{"CREATE TABLE IF NOT EXISTS public.posts"},
|
wantContain: []string{"CREATE TABLE IF NOT EXISTS \"public\".\"posts\""},
|
||||||
wantAbsent: []string{"public.users"},
|
wantAbsent: []string{"\"public\".\"users\""},
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: "missing column is added and existing columns are skipped",
|
name: "missing column is added and existing columns are skipped",
|
||||||
@@ -68,7 +68,7 @@ func TestDiffStatements(t *testing.T) {
|
|||||||
current: func() *models.Database {
|
current: func() *models.Database {
|
||||||
return diffTestDB("public", map[string]map[string]string{"users": {"id": "integer"}})
|
return diffTestDB("public", map[string]map[string]string{"users": {"id": "integer"}})
|
||||||
},
|
},
|
||||||
wantContain: []string{"ADD COLUMN IF NOT EXISTS email text"},
|
wantContain: []string{"ADD COLUMN IF NOT EXISTS \"email\" text"},
|
||||||
wantAbsent: []string{"COLUMN id"},
|
wantAbsent: []string{"COLUMN id"},
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
@@ -79,7 +79,7 @@ func TestDiffStatements(t *testing.T) {
|
|||||||
current: func() *models.Database {
|
current: func() *models.Database {
|
||||||
return diffTestDB("public", map[string]map[string]string{"users": {"id": "integer"}})
|
return diffTestDB("public", map[string]map[string]string{"users": {"id": "integer"}})
|
||||||
},
|
},
|
||||||
wantContain: []string{"ALTER COLUMN id TYPE bigint"},
|
wantContain: []string{"ALTER COLUMN \"id\" TYPE bigint"},
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: "missing non-public schema is created before its tables",
|
name: "missing non-public schema is created before its tables",
|
||||||
@@ -88,8 +88,8 @@ func TestDiffStatements(t *testing.T) {
|
|||||||
},
|
},
|
||||||
current: func() *models.Database { return models.InitDatabase("testdb") },
|
current: func() *models.Database { return models.InitDatabase("testdb") },
|
||||||
wantContain: []string{
|
wantContain: []string{
|
||||||
"CREATE SCHEMA IF NOT EXISTS app",
|
"CREATE SCHEMA IF NOT EXISTS \"app\"",
|
||||||
"CREATE TABLE IF NOT EXISTS app.users",
|
"CREATE TABLE IF NOT EXISTS \"app\".\"users\"",
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
@@ -149,7 +149,7 @@ func TestDiffStatements_NewTableCreatesPrimaryKeySequence(t *testing.T) {
|
|||||||
id := models.InitColumn("id", "users", "public")
|
id := models.InitColumn("id", "users", "public")
|
||||||
id.Type = "integer"
|
id.Type = "integer"
|
||||||
id.IsPrimaryKey = true
|
id.IsPrimaryKey = true
|
||||||
id.Default = "nextval('public.users_id_seq'::regclass)"
|
id.Default = "nextval('\"public\".\"users_id_seq\"'::regclass)"
|
||||||
table.Columns["id"] = id
|
table.Columns["id"] = id
|
||||||
schema.Tables = append(schema.Tables, table)
|
schema.Tables = append(schema.Tables, table)
|
||||||
model.Schemas = append(model.Schemas, schema)
|
model.Schemas = append(model.Schemas, schema)
|
||||||
@@ -160,7 +160,7 @@ func TestDiffStatements_NewTableCreatesPrimaryKeySequence(t *testing.T) {
|
|||||||
t.Fatalf("diffStatements failed: %v", err)
|
t.Fatalf("diffStatements failed: %v", err)
|
||||||
}
|
}
|
||||||
out := diffJoin(stmts)
|
out := diffJoin(stmts)
|
||||||
if !strings.Contains(out, "CREATE SEQUENCE IF NOT EXISTS public.users_id_seq") {
|
if !strings.Contains(out, "CREATE SEQUENCE IF NOT EXISTS \"public\".\"users_id_seq\"") {
|
||||||
t.Fatalf("expected sequence creation, got:\n%s", out)
|
t.Fatalf("expected sequence creation, got:\n%s", out)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -193,7 +193,7 @@ func TestDiffStatements_NewIndexAndChangedIndexRecreated(t *testing.T) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("diffStatements failed: %v", err)
|
t.Fatalf("diffStatements failed: %v", err)
|
||||||
}
|
}
|
||||||
if !strings.Contains(diffJoin(stmts), "CREATE INDEX IF NOT EXISTS idx_users_lookup") {
|
if !strings.Contains(diffJoin(stmts), "CREATE INDEX IF NOT EXISTS \"idx_users_lookup\"") {
|
||||||
t.Fatalf("expected index creation, got:\n%s", diffJoin(stmts))
|
t.Fatalf("expected index creation, got:\n%s", diffJoin(stmts))
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -329,7 +329,7 @@ func TestDiffStatements_ChangedPrimaryKeyIsRecreated(t *testing.T) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("diffStatements failed: %v", err)
|
t.Fatalf("diffStatements failed: %v", err)
|
||||||
}
|
}
|
||||||
if !strings.Contains(diffJoin(stmts), "DROP CONSTRAINT IF EXISTS pk_public_users") {
|
if !strings.Contains(diffJoin(stmts), "DROP CONSTRAINT IF EXISTS \"pk_public_users\"") {
|
||||||
t.Fatalf("expected old primary key to be dropped, got:\n%s", diffJoin(stmts))
|
t.Fatalf("expected old primary key to be dropped, got:\n%s", diffJoin(stmts))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -345,7 +345,7 @@ func TestDiffStatements_ChangedCommentEmitted(t *testing.T) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("diffStatements failed: %v", err)
|
t.Fatalf("diffStatements failed: %v", err)
|
||||||
}
|
}
|
||||||
if !strings.Contains(diffJoin(stmts), "COMMENT ON TABLE public.users IS 'new'") {
|
if !strings.Contains(diffJoin(stmts), "COMMENT ON TABLE \"public\".\"users\" IS 'new'") {
|
||||||
t.Fatalf("expected changed comment, got:\n%s", diffJoin(stmts))
|
t.Fatalf("expected changed comment, got:\n%s", diffJoin(stmts))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -55,7 +55,7 @@ func TestPgDirectives_WriteDatabasePath(t *testing.T) {
|
|||||||
for _, want := range []string{
|
for _, want := range []string{
|
||||||
") PARTITION BY RANGE (created_at) TABLESPACE fast_data",
|
") PARTITION BY RANGE (created_at) TABLESPACE fast_data",
|
||||||
"GENERATED ALWAYS AS IDENTITY",
|
"GENERATED ALWAYS AS IDENTITY",
|
||||||
"WITH (fillfactor=90) TABLESPACE idx_space",
|
"WITH (fillfactor=90) TABLESPACE \"idx_space\"",
|
||||||
} {
|
} {
|
||||||
if !strings.Contains(out, want) {
|
if !strings.Contains(out, want) {
|
||||||
t.Errorf("missing %q in:\n%s", want, out)
|
t.Errorf("missing %q in:\n%s", want, out)
|
||||||
@@ -78,7 +78,7 @@ func TestPgDirectives_WriteSchemaPath(t *testing.T) {
|
|||||||
if !strings.Contains(out, ") PARTITION BY RANGE (created_at) TABLESPACE fast_data;") {
|
if !strings.Contains(out, ") PARTITION BY RANGE (created_at) TABLESPACE fast_data;") {
|
||||||
t.Errorf("table suffix missing from WriteSchema path:\n%s", out)
|
t.Errorf("table suffix missing from WriteSchema path:\n%s", out)
|
||||||
}
|
}
|
||||||
if !strings.Contains(out, "WITH (fillfactor=90) TABLESPACE idx_space") {
|
if !strings.Contains(out, "WITH (fillfactor=90) TABLESPACE \"idx_space\"") {
|
||||||
t.Errorf("index clauses missing from WriteSchema path:\n%s", out)
|
t.Errorf("index clauses missing from WriteSchema path:\n%s", out)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -88,8 +88,8 @@ func TestGeneratedColumns_FullDDL(t *testing.T) {
|
|||||||
}
|
}
|
||||||
out := diffJoin(stmts)
|
out := diffJoin(stmts)
|
||||||
for _, s := range []string{
|
for _, s := range []string{
|
||||||
"full_name text GENERATED ALWAYS AS (first || ' ' || last) STORED",
|
"\"full_name\" text GENERATED ALWAYS AS (first || ' ' || last) STORED",
|
||||||
"seq bigint GENERATED ALWAYS AS IDENTITY NOT NULL",
|
"\"seq\" bigint GENERATED ALWAYS AS IDENTITY NOT NULL",
|
||||||
} {
|
} {
|
||||||
if !strings.Contains(out, s) {
|
if !strings.Contains(out, s) {
|
||||||
t.Errorf("missing %q:\n%s", s, out)
|
t.Errorf("missing %q:\n%s", s, out)
|
||||||
|
|||||||
@@ -0,0 +1,116 @@
|
|||||||
|
package pgsql
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"reflect"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"git.warky.dev/wdevs/relspecgo/pkg/models"
|
||||||
|
"git.warky.dev/wdevs/relspecgo/pkg/transform"
|
||||||
|
"git.warky.dev/wdevs/relspecgo/pkg/writers"
|
||||||
|
)
|
||||||
|
|
||||||
|
// prepareModel rejects NUL before creating files or executing SQL. SQL names
|
||||||
|
// cannot contain NUL; pgx's identifier sanitizer would otherwise remove it.
|
||||||
|
func prepareModel(db *models.Database, lower bool) (*models.Database, error) {
|
||||||
|
if db == nil {
|
||||||
|
return nil, fmt.Errorf("database is required")
|
||||||
|
}
|
||||||
|
seen := map[any]bool{}
|
||||||
|
var validate func(reflect.Value) error
|
||||||
|
validate = func(v reflect.Value) error {
|
||||||
|
switch v.Kind() {
|
||||||
|
case reflect.Pointer:
|
||||||
|
if v.IsNil() || v.Type().Elem().PkgPath() != reflect.TypeOf(models.Database{}).PkgPath() {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if seen[v.Interface()] {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
seen[v.Interface()] = true
|
||||||
|
return validate(v.Elem())
|
||||||
|
case reflect.Struct:
|
||||||
|
for i := 0; i < v.NumField(); i++ {
|
||||||
|
if v.Type().Field(i).Name == "Metadata" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if err := validate(v.Field(i)); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
case reflect.String:
|
||||||
|
if strings.ContainsRune(v.String(), 0) {
|
||||||
|
return fmt.Errorf("PostgreSQL model contains a NUL byte")
|
||||||
|
}
|
||||||
|
case reflect.Slice:
|
||||||
|
for i := 0; i < v.Len(); i++ {
|
||||||
|
if err := validate(v.Index(i)); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
case reflect.Map:
|
||||||
|
if v.Type().Elem().Kind() == reflect.Pointer {
|
||||||
|
iter := v.MapRange()
|
||||||
|
for iter.Next() {
|
||||||
|
if err := validate(iter.Value()); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if err := validate(reflect.ValueOf(db)); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if lower {
|
||||||
|
return transform.LowercaseIdentifiers(db)
|
||||||
|
}
|
||||||
|
return db, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (w *Writer) prepareSchema(schema *models.Schema) (*models.Schema, error) {
|
||||||
|
db, err := prepareModel(&models.Database{Schemas: []*models.Schema{schema}}, w.options.LowercaseIdentifiers)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return db.Schemas[0], nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// regclassLastName decodes quoted identifiers in a qualified regclass spelling.
|
||||||
|
func regclassLastName(text string) string {
|
||||||
|
var name strings.Builder
|
||||||
|
quoted := false
|
||||||
|
for i := 0; i < len(text); i++ {
|
||||||
|
c := text[i]
|
||||||
|
if c == '"' {
|
||||||
|
if quoted && i+1 < len(text) && text[i+1] == '"' {
|
||||||
|
name.WriteByte('"')
|
||||||
|
i++
|
||||||
|
} else {
|
||||||
|
quoted = !quoted
|
||||||
|
}
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if c == '.' && !quoted {
|
||||||
|
name.Reset()
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
name.WriteByte(c)
|
||||||
|
}
|
||||||
|
return name.String()
|
||||||
|
}
|
||||||
|
|
||||||
|
// quoteDefaultValue keeps explicit SQL expressions intact and uses the same
|
||||||
|
// backslash-safe literal quoting as other PostgreSQL value contexts.
|
||||||
|
func quoteDefaultValue(value, sqlType string) string {
|
||||||
|
trimmed := strings.TrimSpace(value)
|
||||||
|
if len(trimmed) >= 3 && (trimmed[0] == 'E' || trimmed[0] == 'e') && trimmed[1] == '\'' && trimmed[len(trimmed)-1] == '\'' {
|
||||||
|
return trimmed
|
||||||
|
}
|
||||||
|
formatted := writers.QuoteDefaultValue(value, sqlType)
|
||||||
|
if formatted == "'"+strings.ReplaceAll(trimmed, "'", "''")+"'" {
|
||||||
|
return quote(trimmed)
|
||||||
|
}
|
||||||
|
return formatted
|
||||||
|
}
|
||||||
@@ -3,9 +3,11 @@ package pgsql
|
|||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
@@ -141,15 +143,16 @@ func TestLive_WriteDatabaseEmptyThenIdenticalThenDrifted(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestLive_WriteDatabaseFailedStatementContinues(t *testing.T) {
|
func TestLive_WriteDatabaseFailedStatementReturnsError(t *testing.T) {
|
||||||
connString := liveWriterConn(t)
|
connString := liveWriterConn(t)
|
||||||
schema, _ := liveWriterSchema(t, connString)
|
schema, _ := liveWriterSchema(t, connString)
|
||||||
reportPath := filepath.Join(t.TempDir(), "report.json")
|
reportPath := filepath.Join(t.TempDir(), "report.json")
|
||||||
|
|
||||||
db := liveModel(schema, map[string]string{"bad": "no_such_type_xyz"})
|
db := liveModel(schema, map[string]string{"bad": "no_such_type_xyz"})
|
||||||
rep, err := runLiveWrite(t, connString, db, map[string]interface{}{"full_ddl": true, "report_path": reportPath})
|
rep, err := runLiveWrite(t, connString, db, map[string]interface{}{"full_ddl": true, "report_path": reportPath})
|
||||||
if err != nil {
|
var failure *ExecutionFailure
|
||||||
t.Fatalf("failed statements must not abort the run: %v", err)
|
if !errors.As(err, &failure) {
|
||||||
|
t.Fatalf("expected structured failure, got: %v", err)
|
||||||
}
|
}
|
||||||
if rep.FailedStatements == 0 || len(rep.Errors) != rep.FailedStatements {
|
if rep.FailedStatements == 0 || len(rep.Errors) != rep.FailedStatements {
|
||||||
t.Fatalf("expected recorded failures: %+v", rep)
|
t.Fatalf("expected recorded failures: %+v", rep)
|
||||||
@@ -230,7 +233,7 @@ func TestLive_ExecuteStatementsSkipsCommentsAndBlank(t *testing.T) {
|
|||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
r := w.executionReport
|
r := w.executionReport
|
||||||
if r.ExecutedStatements != 2 || r.FailedStatements != 0 || r.TotalStatements != 5 {
|
if r.ExecutedStatements != 2 || r.FailedStatements != 0 || r.TotalStatements != 5 || r.SkippedStatements != 3 {
|
||||||
t.Errorf("counts: %+v", r)
|
t.Errorf("counts: %+v", r)
|
||||||
}
|
}
|
||||||
if len(r.Schemas) != 1 || r.Schemas[0].Name != schema || len(r.Schemas[0].Tables) != 1 || !r.Schemas[0].Tables[0].Created {
|
if len(r.Schemas) != 1 || r.Schemas[0].Name != schema || len(r.Schemas[0].Tables) != 1 || !r.Schemas[0].Tables[0].Created {
|
||||||
@@ -241,3 +244,174 @@ func TestLive_ExecuteStatementsSkipsCommentsAndBlank(t *testing.T) {
|
|||||||
t.Errorf("table not created: %v", err)
|
t.Errorf("table not created: %v", err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestLive_ExecuteStatementsFailureModes(t *testing.T) {
|
||||||
|
connString := liveWriterConn(t)
|
||||||
|
for _, continueOnError := range []bool{false, true} {
|
||||||
|
t.Run(fmt.Sprintf("continue=%v", continueOnError), func(t *testing.T) {
|
||||||
|
schema, conn := liveWriterSchema(t, connString)
|
||||||
|
reportPath := filepath.Join(t.TempDir(), "report.json")
|
||||||
|
w := NewWriter(&writers.WriterOptions{
|
||||||
|
ContinueOnError: continueOnError,
|
||||||
|
Metadata: map[string]interface{}{"report_path": reportPath},
|
||||||
|
})
|
||||||
|
w.executionReport = &ExecutionReport{}
|
||||||
|
err := w.executeStatements([]string{
|
||||||
|
"CREATE SCHEMA " + schema,
|
||||||
|
"SELECT 1 / 0",
|
||||||
|
"CREATE TABLE " + schema + ".after_failure (id int)",
|
||||||
|
}, connString)
|
||||||
|
var failure *ExecutionFailure
|
||||||
|
if !errors.As(err, &failure) || failure.Report != w.executionReport {
|
||||||
|
t.Fatalf("expected structured execution failure, got %v", err)
|
||||||
|
}
|
||||||
|
var report ExecutionReport
|
||||||
|
data, err := os.ReadFile(reportPath)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := json.Unmarshal(data, &report); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
executed, skipped := 1, 1
|
||||||
|
if continueOnError {
|
||||||
|
executed, skipped = 2, 0
|
||||||
|
}
|
||||||
|
if report.TotalStatements != 3 || report.ExecutedStatements != executed || report.FailedStatements != 1 || report.SkippedStatements != skipped || report.EndTime == "" {
|
||||||
|
t.Fatalf("incorrect partial application report: %+v", report)
|
||||||
|
}
|
||||||
|
if len(report.Errors) != 1 || report.Errors[0].StatementNumber != 2 {
|
||||||
|
t.Fatalf("incorrect failure details: %+v", report.Errors)
|
||||||
|
}
|
||||||
|
if exists := columnExists(t, conn, schema, "after_failure", "id"); exists != continueOnError {
|
||||||
|
t.Fatalf("later statement executed=%v, continue=%v", exists, continueOnError)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestLive_IncompatibleTypeChangePreservesColumnAndData(t *testing.T) {
|
||||||
|
connString := liveWriterConn(t)
|
||||||
|
schema, conn := liveWriterSchema(t, connString)
|
||||||
|
ctx := context.Background()
|
||||||
|
_, err := conn.Exec(ctx, "CREATE SCHEMA "+schema+"; CREATE TABLE "+schema+".accounts (id integer PRIMARY KEY, name text UNIQUE); INSERT INTO "+schema+".accounts VALUES (1, 'not-an-integer')")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
var originalIndex uint32
|
||||||
|
if err := conn.QueryRow(ctx, "SELECT to_regclass($1)::oid", schema+".accounts_name_key").Scan(&originalIndex); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Exercise both full-DDL and diff generation against populated data.
|
||||||
|
for _, fullDDL := range []bool{false, true} {
|
||||||
|
t.Run(fmt.Sprintf("full_ddl=%v", fullDDL), func(t *testing.T) {
|
||||||
|
model := liveModel(schema, map[string]string{"name": "integer"})
|
||||||
|
unique := models.InitConstraint("accounts_name_key", models.UniqueConstraint)
|
||||||
|
unique.Columns = []string{"name"}
|
||||||
|
model.Schemas[0].Tables[0].Constraints[unique.Name] = unique
|
||||||
|
report, err := runLiveWrite(t, connString, model, map[string]interface{}{"full_ddl": fullDDL})
|
||||||
|
var failure *ExecutionFailure
|
||||||
|
if !errors.As(err, &failure) || report.FailedStatements != 1 {
|
||||||
|
t.Fatalf("expected failed conversion, got report=%+v err=%v", report, err)
|
||||||
|
}
|
||||||
|
var value, typ string
|
||||||
|
if err := conn.QueryRow(ctx, "SELECT name, pg_typeof(name)::text FROM "+schema+".accounts WHERE id=1").Scan(&value, &typ); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if value != "not-an-integer" || typ != "text" {
|
||||||
|
t.Fatalf("original column changed: value=%q type=%q", value, typ)
|
||||||
|
}
|
||||||
|
var columns int
|
||||||
|
if err := conn.QueryRow(ctx, "SELECT count(*) FROM information_schema.columns WHERE table_schema=$1 AND table_name='accounts'", schema).Scan(&columns); err != nil || columns != 2 {
|
||||||
|
t.Fatalf("unexpected replacement column: columns=%d err=%v", columns, err)
|
||||||
|
}
|
||||||
|
var index uint32
|
||||||
|
if err := conn.QueryRow(ctx, "SELECT to_regclass($1)::oid", schema+".accounts_name_key").Scan(&index); err != nil || index != originalIndex {
|
||||||
|
t.Fatalf("unique index changed: index=%d original=%d err=%v", index, originalIndex, err)
|
||||||
|
}
|
||||||
|
if _, err := conn.Exec(ctx, "INSERT INTO "+schema+".accounts VALUES (2, 'not-an-integer')"); err == nil {
|
||||||
|
t.Fatal("unique constraint no longer enforced")
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestLive_AtomicExecutionRollback(t *testing.T) {
|
||||||
|
connString := liveWriterConn(t)
|
||||||
|
schema, conn := liveWriterSchema(t, connString)
|
||||||
|
w := NewWriter(&writers.WriterOptions{AtomicExecution: true})
|
||||||
|
w.executionReport = &ExecutionReport{}
|
||||||
|
err := w.executeStatements([]string{
|
||||||
|
"-- Schema: " + schema,
|
||||||
|
"CREATE SCHEMA " + schema,
|
||||||
|
"CREATE TABLE " + schema + ".created (id int)",
|
||||||
|
"SELECT 1 / 0",
|
||||||
|
"SELECT 1",
|
||||||
|
}, connString)
|
||||||
|
var failure *ExecutionFailure
|
||||||
|
if !errors.As(err, &failure) {
|
||||||
|
t.Fatalf("expected execution failure: %v", err)
|
||||||
|
}
|
||||||
|
report := failure.Report
|
||||||
|
if report.Outcome != "rolled_back" || report.RolledBackStatements != 2 || report.ExecutedStatements != 0 || report.FailedStatements != 1 || report.SkippedStatements != 2 {
|
||||||
|
t.Fatalf("rollback report: %+v", report)
|
||||||
|
}
|
||||||
|
if report.Schemas[0].Tables[0].Created {
|
||||||
|
t.Fatal("rolled-back table must not be reported created")
|
||||||
|
}
|
||||||
|
if columnExists(t, conn, schema, "created", "id") {
|
||||||
|
t.Fatal("table survived transaction rollback")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestLive_TypePreflightPrecedesDropsEvenWithContinuation(t *testing.T) {
|
||||||
|
connString := liveWriterConn(t)
|
||||||
|
schema, conn := liveWriterSchema(t, connString)
|
||||||
|
_, err := conn.Exec(context.Background(), "CREATE SCHEMA "+schema+"; CREATE TABLE "+schema+".accounts (id integer PRIMARY KEY, name text UNIQUE); INSERT INTO "+schema+".accounts VALUES (1, '123'), (2, 'bad')")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
// The desired model would drop the unique constraint. Incompatible data must
|
||||||
|
// be caught first, including bad data after the first row.
|
||||||
|
model := liveModel(schema, map[string]string{"name": "integer"})
|
||||||
|
w := NewWriter(&writers.WriterOptions{ContinueOnError: true, Metadata: map[string]interface{}{"connection_string": connString}})
|
||||||
|
err = w.WriteDatabase(model)
|
||||||
|
var failure *ExecutionFailure
|
||||||
|
if !errors.As(err, &failure) || failure.Report.FailedStatements != 1 || failure.Report.ExecutedStatements != 0 {
|
||||||
|
t.Fatalf("preflight must stop mutations: %+v %v", w.executionReport, err)
|
||||||
|
}
|
||||||
|
var exists bool
|
||||||
|
if err := conn.QueryRow(context.Background(), "SELECT to_regclass($1) IS NOT NULL", schema+".accounts_name_key").Scan(&exists); err != nil || !exists {
|
||||||
|
t.Fatalf("constraint was dropped before preflight: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestLive_CustomTypeConversionAndPredicate(t *testing.T) {
|
||||||
|
connString := liveWriterConn(t)
|
||||||
|
for _, fullDDL := range []bool{false, true} {
|
||||||
|
t.Run(fmt.Sprintf("full_ddl=%v", fullDDL), func(t *testing.T) {
|
||||||
|
schema, conn := liveWriterSchema(t, connString)
|
||||||
|
_, err := conn.Exec(context.Background(), "CREATE SCHEMA "+schema+"; CREATE TABLE "+schema+".accounts (id integer PRIMARY KEY, name text); INSERT INTO "+schema+".accounts VALUES (1,'42 units'),(2,'7 units')")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
model := liveModel(schema, map[string]string{"name": "integer"})
|
||||||
|
col := model.Schemas[0].Tables[0].Columns["name"]
|
||||||
|
col.Metadata["pgsql_using"] = "split_part(name, ' ', 1)::integer"
|
||||||
|
col.Metadata["pgsql_preflight"] = "name ~ '^[0-9]+ units$' AND name NOT LIKE '7 %'"
|
||||||
|
meta := map[string]interface{}{"full_ddl": fullDDL}
|
||||||
|
if _, err := runLiveWrite(t, connString, model, meta); err == nil || !strings.Contains(err.Error(), "preflight rejected rows") {
|
||||||
|
t.Fatalf("predicate must reject: %v", err)
|
||||||
|
}
|
||||||
|
col.Metadata["pgsql_preflight"] = "name ~ '^[0-9]+ units$'"
|
||||||
|
if _, err := runLiveWrite(t, connString, model, meta); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
var value int
|
||||||
|
if err := conn.QueryRow(context.Background(), "SELECT sum(name) FROM "+schema+".accounts").Scan(&value); err != nil || value != 49 {
|
||||||
|
t.Fatalf("custom conversion result %d: %v", value, err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -61,6 +61,16 @@ func (w *MigrationWriter) GenerateScripts(model, current *models.Database) ([]Mi
|
|||||||
current = models.InitDatabase(model.Name)
|
current = models.InitDatabase(model.Name)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
var err error
|
||||||
|
model, err = prepareModel(model, w.options.LowercaseIdentifiers)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
current, err = prepareModel(current, false)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
// Check if audit is configured in metadata
|
// Check if audit is configured in metadata
|
||||||
var auditConfig *AuditConfig
|
var auditConfig *AuditConfig
|
||||||
if w.options.Metadata != nil {
|
if w.options.Metadata != nil {
|
||||||
@@ -93,7 +103,7 @@ func (w *MigrationWriter) GenerateScripts(model, current *models.Database) ([]Mi
|
|||||||
if cs == nil {
|
if cs == nil {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
if strings.EqualFold(cs.Name, modelSchema.Name) {
|
if cs.Name == modelSchema.Name {
|
||||||
currentSchema = cs
|
currentSchema = cs
|
||||||
break
|
break
|
||||||
}
|
}
|
||||||
@@ -166,7 +176,7 @@ func (w *MigrationWriter) WriteMigration(model, current *models.Database) error
|
|||||||
// Write header
|
// Write header
|
||||||
fmt.Fprintf(w.writer, "-- PostgreSQL Migration Script\n")
|
fmt.Fprintf(w.writer, "-- PostgreSQL Migration Script\n")
|
||||||
fmt.Fprintf(w.writer, "-- Generated by RelSpec\n")
|
fmt.Fprintf(w.writer, "-- Generated by RelSpec\n")
|
||||||
fmt.Fprintf(w.writer, "-- Source: %s -> %s\n", current.Name, model.Name)
|
fmt.Fprintf(w.writer, "-- Source: %s -> %s\n", commentText(current.Name), commentText(model.Name))
|
||||||
if w.options.ContinueOnError {
|
if w.options.ContinueOnError {
|
||||||
fmt.Fprintf(w.writer, "\\set ON_ERROR_STOP off\n")
|
fmt.Fprintf(w.writer, "\\set ON_ERROR_STOP off\n")
|
||||||
}
|
}
|
||||||
@@ -175,7 +185,7 @@ func (w *MigrationWriter) WriteMigration(model, current *models.Database) error
|
|||||||
// Write scripts
|
// Write scripts
|
||||||
for _, script := range scripts {
|
for _, script := range scripts {
|
||||||
fmt.Fprintf(w.writer, "-- Priority: %d | Type: %s | Object: %s\n",
|
fmt.Fprintf(w.writer, "-- Priority: %d | Type: %s | Object: %s\n",
|
||||||
script.Priority, script.ObjectType, script.ObjectName)
|
script.Priority, commentText(script.ObjectType), commentText(script.ObjectName))
|
||||||
fmt.Fprintf(w.writer, "%s\n\n", script.Body)
|
fmt.Fprintf(w.writer, "%s\n\n", script.Body)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -256,12 +266,12 @@ func (w *MigrationWriter) generateDropScripts(model, current *models.Schema) ([]
|
|||||||
// Build map of model tables for quick lookup
|
// Build map of model tables for quick lookup
|
||||||
modelTables := make(map[string]*models.Table)
|
modelTables := make(map[string]*models.Table)
|
||||||
for _, table := range model.Tables {
|
for _, table := range model.Tables {
|
||||||
modelTables[strings.ToLower(table.Name)] = table
|
modelTables[table.Name] = table
|
||||||
}
|
}
|
||||||
|
|
||||||
// Find constraints to drop
|
// Find constraints to drop
|
||||||
for _, currentTable := range current.Tables {
|
for _, currentTable := range current.Tables {
|
||||||
modelTable, existsInModel := modelTables[strings.ToLower(currentTable.Name)]
|
modelTable, existsInModel := modelTables[currentTable.Name]
|
||||||
|
|
||||||
if !existsInModel {
|
if !existsInModel {
|
||||||
continue
|
continue
|
||||||
@@ -296,7 +306,7 @@ func (w *MigrationWriter) generateDropScripts(model, current *models.Schema) ([]
|
|||||||
if refSchema == "" {
|
if refSchema == "" {
|
||||||
refSchema = current.Name
|
refSchema = current.Name
|
||||||
}
|
}
|
||||||
if strings.EqualFold(refTable, currentTable.Name) && strings.EqualFold(refSchema, current.Name) {
|
if (refTable == currentTable.Name) && (refSchema == current.Name) {
|
||||||
fkKey := fmt.Sprintf("%s.%s.%s", current.Name, otherTable.Name, fkName)
|
fkKey := fmt.Sprintf("%s.%s.%s", current.Name, otherTable.Name, fkName)
|
||||||
if !droppedFKs[fkKey] {
|
if !droppedFKs[fkKey] {
|
||||||
droppedFKs[fkKey] = true
|
droppedFKs[fkKey] = true
|
||||||
@@ -395,13 +405,13 @@ func (w *MigrationWriter) generateTableScripts(model, current *models.Schema) ([
|
|||||||
currentTables := make(map[string]*models.Table)
|
currentTables := make(map[string]*models.Table)
|
||||||
if current != nil {
|
if current != nil {
|
||||||
for _, table := range current.Tables {
|
for _, table := range current.Tables {
|
||||||
currentTables[strings.ToLower(table.Name)] = table
|
currentTables[table.Name] = table
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Process each model table
|
// Process each model table
|
||||||
for _, modelTable := range model.Tables {
|
for _, modelTable := range model.Tables {
|
||||||
currentTable, exists := currentTables[strings.ToLower(modelTable.Name)]
|
currentTable, exists := currentTables[modelTable.Name]
|
||||||
|
|
||||||
if !exists {
|
if !exists {
|
||||||
// Table doesn't exist, create it
|
// Table doesn't exist, create it
|
||||||
@@ -439,12 +449,12 @@ func (w *MigrationWriter) generateAlterTableScripts(schema *models.Schema, model
|
|||||||
// Build map of current columns
|
// Build map of current columns
|
||||||
currentColumns := make(map[string]*models.Column)
|
currentColumns := make(map[string]*models.Column)
|
||||||
for name, col := range currentTable.Columns {
|
for name, col := range currentTable.Columns {
|
||||||
currentColumns[strings.ToLower(name)] = col
|
currentColumns[name] = col
|
||||||
}
|
}
|
||||||
|
|
||||||
// Check each model column
|
// Check each model column
|
||||||
for _, modelCol := range sortColumns(modelTable.Columns) {
|
for _, modelCol := range sortColumns(modelTable.Columns) {
|
||||||
currentCol, exists := currentColumns[strings.ToLower(modelCol.Name)]
|
currentCol, exists := currentColumns[modelCol.Name]
|
||||||
|
|
||||||
if !exists {
|
if !exists {
|
||||||
// Column doesn't exist, add it
|
// Column doesn't exist, add it
|
||||||
@@ -487,15 +497,16 @@ func (w *MigrationWriter) generateAlterTableScripts(schema *models.Schema, model
|
|||||||
} else if !columnsEqual(modelCol, currentCol) {
|
} else if !columnsEqual(modelCol, currentCol) {
|
||||||
// Column exists but properties changed
|
// Column exists but properties changed
|
||||||
if !columnTypesEqual(modelCol, currentCol) {
|
if !columnTypesEqual(modelCol, currentCol) {
|
||||||
newType := effectiveAlterColumnSQLType(modelCol)
|
data, err := conversionData(schema.Name, modelTable.Name, modelCol)
|
||||||
sql, err := w.executor.ExecuteAlterColumnTypeWithCheck(AlterColumnTypeWithCheckData{
|
if err != nil {
|
||||||
SchemaName: schema.Name,
|
return nil, err
|
||||||
TableName: modelTable.Name,
|
}
|
||||||
ColumnName: modelCol.Name,
|
scripts = append(scripts, MigrationScript{
|
||||||
NewType: newType,
|
ObjectName: fmt.Sprintf("%s.%s.%s", schema.Name, modelTable.Name, modelCol.Name),
|
||||||
EquivalentTypes: equivalentTypeListSQL(newType),
|
ObjectType: "preflight column type", Schema: schema.Name, Priority: -10000,
|
||||||
UsingExpr: buildAlterColumnUsingExpression(modelCol.Name, newType),
|
Sequence: len(scripts), Body: preflightColumnType(data, w.options.FlattenSchema),
|
||||||
})
|
})
|
||||||
|
sql, err := w.executor.ExecuteAlterColumnTypeWithCheck(data)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
@@ -573,13 +584,13 @@ func (w *MigrationWriter) generateIndexScripts(model, current *models.Schema) ([
|
|||||||
currentTables := make(map[string]*models.Table)
|
currentTables := make(map[string]*models.Table)
|
||||||
if current != nil {
|
if current != nil {
|
||||||
for _, table := range current.Tables {
|
for _, table := range current.Tables {
|
||||||
currentTables[strings.ToLower(table.Name)] = table
|
currentTables[table.Name] = table
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Process each model table
|
// Process each model table
|
||||||
for _, modelTable := range model.Tables {
|
for _, modelTable := range model.Tables {
|
||||||
currentTable := currentTables[strings.ToLower(modelTable.Name)]
|
currentTable := currentTables[modelTable.Name]
|
||||||
|
|
||||||
// Process primary keys first - check explicit constraints
|
// Process primary keys first - check explicit constraints
|
||||||
foundExplicitPK := false
|
foundExplicitPK := false
|
||||||
@@ -602,7 +613,7 @@ func (w *MigrationWriter) generateIndexScripts(model, current *models.Schema) ([
|
|||||||
SchemaName: model.Name,
|
SchemaName: model.Name,
|
||||||
TableName: modelTable.Name,
|
TableName: modelTable.Name,
|
||||||
ConstraintName: constraintName,
|
ConstraintName: constraintName,
|
||||||
Columns: strings.Join(constraint.Columns, ", "),
|
Columns: quotedIdentifiers(constraint.Columns),
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
@@ -626,12 +637,12 @@ func (w *MigrationWriter) generateIndexScripts(model, current *models.Schema) ([
|
|||||||
pkColumns := []string{}
|
pkColumns := []string{}
|
||||||
for _, col := range modelTable.Columns {
|
for _, col := range modelTable.Columns {
|
||||||
if col.IsPrimaryKey {
|
if col.IsPrimaryKey {
|
||||||
pkColumns = append(pkColumns, col.SQLName())
|
pkColumns = append(pkColumns, col.Name)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if len(pkColumns) > 0 {
|
if len(pkColumns) > 0 {
|
||||||
sort.Strings(pkColumns)
|
sort.Strings(pkColumns)
|
||||||
constraintName := fmt.Sprintf("pk_%s_%s", model.SQLName(), modelTable.SQLName())
|
constraintName := fmt.Sprintf("pk_%s_%s", model.Name, modelTable.Name)
|
||||||
shouldCreate := true
|
shouldCreate := true
|
||||||
|
|
||||||
if currentTable != nil {
|
if currentTable != nil {
|
||||||
@@ -649,7 +660,7 @@ func (w *MigrationWriter) generateIndexScripts(model, current *models.Schema) ([
|
|||||||
SchemaName: model.Name,
|
SchemaName: model.Name,
|
||||||
TableName: modelTable.Name,
|
TableName: modelTable.Name,
|
||||||
ConstraintName: constraintName,
|
ConstraintName: constraintName,
|
||||||
Columns: strings.Join(pkColumns, ", "),
|
Columns: quotedIdentifiers(pkColumns),
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
@@ -749,7 +760,7 @@ func buildIndexColumnExpressionsFiltered(table *models.Table, index *models.Inde
|
|||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
colExpr := col.SQLName()
|
colExpr := quoteIdent(col.Name)
|
||||||
if opClass := indexOperatorClassForColumn(col, indexType, index.Comment); opClass != "" {
|
if opClass := indexOperatorClassForColumn(col, indexType, index.Comment); opClass != "" {
|
||||||
colExpr = fmt.Sprintf("%s %s", colExpr, opClass)
|
colExpr = fmt.Sprintf("%s %s", colExpr, opClass)
|
||||||
}
|
}
|
||||||
@@ -766,12 +777,12 @@ func (w *MigrationWriter) generateUniqueConstraintScripts(model, current *models
|
|||||||
currentTables := make(map[string]*models.Table)
|
currentTables := make(map[string]*models.Table)
|
||||||
if current != nil {
|
if current != nil {
|
||||||
for _, table := range current.Tables {
|
for _, table := range current.Tables {
|
||||||
currentTables[strings.ToLower(table.Name)] = table
|
currentTables[table.Name] = table
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, modelTable := range model.Tables {
|
for _, modelTable := range model.Tables {
|
||||||
currentTable := currentTables[strings.ToLower(modelTable.Name)]
|
currentTable := currentTables[modelTable.Name]
|
||||||
|
|
||||||
for _, constraint := range sortConstraints(modelTable.Constraints) {
|
for _, constraint := range sortConstraints(modelTable.Constraints) {
|
||||||
if constraint.Type != models.UniqueConstraint {
|
if constraint.Type != models.UniqueConstraint {
|
||||||
@@ -787,7 +798,7 @@ func (w *MigrationWriter) generateUniqueConstraintScripts(model, current *models
|
|||||||
columnExprs := make([]string, 0, len(constraint.Columns))
|
columnExprs := make([]string, 0, len(constraint.Columns))
|
||||||
for _, colName := range constraint.Columns {
|
for _, colName := range constraint.Columns {
|
||||||
if col, ok := modelTable.Columns[colName]; ok {
|
if col, ok := modelTable.Columns[colName]; ok {
|
||||||
columnExprs = append(columnExprs, col.SQLName())
|
columnExprs = append(columnExprs, quoteIdent(col.Name))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if len(columnExprs) == 0 {
|
if len(columnExprs) == 0 {
|
||||||
@@ -837,7 +848,7 @@ func adoptableUniqueIndex(table *models.Table, constraint *models.Constraint) st
|
|||||||
return ""
|
return ""
|
||||||
}
|
}
|
||||||
for i, col := range index.Columns {
|
for i, col := range index.Columns {
|
||||||
if !strings.EqualFold(col, constraint.Columns[i]) {
|
if col != constraint.Columns[i] {
|
||||||
return ""
|
return ""
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -854,13 +865,13 @@ func (w *MigrationWriter) generateForeignKeyScripts(model, current *models.Schem
|
|||||||
currentTables := make(map[string]*models.Table)
|
currentTables := make(map[string]*models.Table)
|
||||||
if current != nil {
|
if current != nil {
|
||||||
for _, table := range current.Tables {
|
for _, table := range current.Tables {
|
||||||
currentTables[strings.ToLower(table.Name)] = table
|
currentTables[table.Name] = table
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Process each model table
|
// Process each model table
|
||||||
for _, modelTable := range model.Tables {
|
for _, modelTable := range model.Tables {
|
||||||
currentTable := currentTables[strings.ToLower(modelTable.Name)]
|
currentTable := currentTables[modelTable.Name]
|
||||||
|
|
||||||
// Process each constraint
|
// Process each constraint
|
||||||
for _, constraint := range sortConstraints(modelTable.Constraints) {
|
for _, constraint := range sortConstraints(modelTable.Constraints) {
|
||||||
@@ -903,10 +914,10 @@ func (w *MigrationWriter) generateForeignKeyScripts(model, current *models.Schem
|
|||||||
SchemaName: model.Name,
|
SchemaName: model.Name,
|
||||||
TableName: modelTable.Name,
|
TableName: modelTable.Name,
|
||||||
ConstraintName: constraintName,
|
ConstraintName: constraintName,
|
||||||
SourceColumns: strings.Join(constraint.Columns, ", "),
|
SourceColumns: quotedIdentifiers(constraint.Columns),
|
||||||
TargetSchema: constraint.ReferencedSchema,
|
TargetSchema: constraint.ReferencedSchema,
|
||||||
TargetTable: constraint.ReferencedTable,
|
TargetTable: constraint.ReferencedTable,
|
||||||
TargetColumns: strings.Join(constraint.ReferencedColumns, ", "),
|
TargetColumns: quotedIdentifiers(constraint.ReferencedColumns),
|
||||||
OnDelete: onDelete,
|
OnDelete: onDelete,
|
||||||
OnUpdate: onUpdate,
|
OnUpdate: onUpdate,
|
||||||
Deferral: deferral,
|
Deferral: deferral,
|
||||||
@@ -938,20 +949,20 @@ func (w *MigrationWriter) generateCommentScripts(model, current *models.Schema)
|
|||||||
currentTables := make(map[string]*models.Table)
|
currentTables := make(map[string]*models.Table)
|
||||||
if current != nil {
|
if current != nil {
|
||||||
for _, table := range current.Tables {
|
for _, table := range current.Tables {
|
||||||
currentTables[strings.ToLower(table.Name)] = table
|
currentTables[table.Name] = table
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Process each model table
|
// Process each model table
|
||||||
for _, modelTable := range model.Tables {
|
for _, modelTable := range model.Tables {
|
||||||
currentTable := currentTables[strings.ToLower(modelTable.Name)]
|
currentTable := currentTables[modelTable.Name]
|
||||||
|
|
||||||
// Table comment (skipped when the live table already carries the same comment)
|
// Table comment (skipped when the live table already carries the same comment)
|
||||||
if modelTable.Description != "" && (currentTable == nil || strings.TrimSpace(currentTable.Description) != strings.TrimSpace(modelTable.Description)) {
|
if modelTable.Description != "" && (currentTable == nil || strings.TrimSpace(currentTable.Description) != strings.TrimSpace(modelTable.Description)) {
|
||||||
sql, err := w.executor.ExecuteCommentTable(CommentTableData{
|
sql, err := w.executor.ExecuteCommentTable(CommentTableData{
|
||||||
SchemaName: model.Name,
|
SchemaName: model.Name,
|
||||||
TableName: modelTable.Name,
|
TableName: modelTable.Name,
|
||||||
Comment: escapeQuote(modelTable.Description),
|
Comment: modelTable.Description,
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
@@ -975,7 +986,7 @@ func (w *MigrationWriter) generateCommentScripts(model, current *models.Schema)
|
|||||||
SchemaName: model.Name,
|
SchemaName: model.Name,
|
||||||
TableName: modelTable.Name,
|
TableName: modelTable.Name,
|
||||||
ColumnName: col.Name,
|
ColumnName: col.Name,
|
||||||
Comment: escapeQuote(col.Description),
|
Comment: col.Description,
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
@@ -1228,21 +1239,21 @@ func constraintsEqual(c1, c2 *models.Constraint) bool {
|
|||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
for i, col := range c1.Columns {
|
for i, col := range c1.Columns {
|
||||||
if !strings.EqualFold(col, c2.Columns[i]) {
|
if col != c2.Columns[i] {
|
||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// For foreign keys, also compare referenced table and columns
|
// For foreign keys, also compare referenced table and columns
|
||||||
if c1.Type == models.ForeignKeyConstraint {
|
if c1.Type == models.ForeignKeyConstraint {
|
||||||
if !strings.EqualFold(c1.ReferencedTable, c2.ReferencedTable) {
|
if c1.ReferencedTable != c2.ReferencedTable {
|
||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
if len(c1.ReferencedColumns) != len(c2.ReferencedColumns) {
|
if len(c1.ReferencedColumns) != len(c2.ReferencedColumns) {
|
||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
for i, col := range c1.ReferencedColumns {
|
for i, col := range c1.ReferencedColumns {
|
||||||
if !strings.EqualFold(col, c2.ReferencedColumns[i]) {
|
if col != c2.ReferencedColumns[i] {
|
||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -1272,7 +1283,7 @@ func indexesEqual(idx1, idx2 *models.Index) bool {
|
|||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
for i, col := range idx1.Columns {
|
for i, col := range idx1.Columns {
|
||||||
if !strings.EqualFold(col, idx2.Columns[i]) {
|
if col != idx2.Columns[i] {
|
||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -1326,7 +1337,7 @@ func currentColumnHasDescription(currentTable *models.Table, col *models.Column)
|
|||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
for _, cc := range currentTable.Columns {
|
for _, cc := range currentTable.Columns {
|
||||||
if strings.EqualFold(cc.Name, col.Name) {
|
if cc.Name == col.Name {
|
||||||
return strings.TrimSpace(cc.Description) == strings.TrimSpace(col.Description)
|
return strings.TrimSpace(cc.Description) == strings.TrimSpace(col.Description)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -1354,7 +1365,7 @@ func primaryKeyColumnsMatch(modelTable *models.Table, current *models.Constraint
|
|||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
for i, col := range modelCols {
|
for i, col := range modelCols {
|
||||||
if !strings.EqualFold(col, current.Columns[i]) {
|
if col != current.Columns[i] {
|
||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -89,7 +89,7 @@ func TestWriteMigration_ArrayDefault(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
output := buf.String()
|
output := buf.String()
|
||||||
if !strings.Contains(output, "tags text[] DEFAULT '{}' NOT NULL") {
|
if !strings.Contains(output, "\"tags\" text[] DEFAULT '{}' NOT NULL") {
|
||||||
t.Fatalf("expected normalized array default in migration, got:\n%s", output)
|
t.Fatalf("expected normalized array default in migration, got:\n%s", output)
|
||||||
}
|
}
|
||||||
if strings.Contains(output, "'''{}'''") {
|
if strings.Contains(output, "'''{}'''") {
|
||||||
@@ -128,15 +128,15 @@ func TestWriteMigration_AltersColumnTypeWhenActualTypeDiffers(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
output := buf.String()
|
output := buf.String()
|
||||||
if !strings.Contains(output, "ALTER TABLE public.learnings") || !strings.Contains(output, "ALTER COLUMN details TYPE text") {
|
if !strings.Contains(output, "ALTER TABLE \"public\".\"learnings\"") || !strings.Contains(output, "ALTER COLUMN \"details\" TYPE text") {
|
||||||
t.Fatalf("expected migration to alter mismatched column type, got:\n%s", output)
|
t.Fatalf("expected migration to alter mismatched column type, got:\n%s", output)
|
||||||
}
|
}
|
||||||
if !strings.Contains(output, `ALTER COLUMN details TYPE text USING details::text;`) {
|
if !strings.Contains(output, `ALTER COLUMN "details" TYPE text USING "details"::text;`) {
|
||||||
t.Fatalf("expected migration type alter to include USING cast, got:\n%s", output)
|
t.Fatalf("expected migration type alter to include USING cast, got:\n%s", output)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestWriteMigration_AltersColumnTypeFallsBackToRenameAndAddOnConversionFailure(t *testing.T) {
|
func TestWriteMigration_AltersColumnTypePreservesConversionFailure(t *testing.T) {
|
||||||
current := models.InitDatabase("testdb")
|
current := models.InitDatabase("testdb")
|
||||||
currentSchema := models.InitSchema("public")
|
currentSchema := models.InitSchema("public")
|
||||||
currentTable := models.InitTable("learnings", "public")
|
currentTable := models.InitTable("learnings", "public")
|
||||||
@@ -167,17 +167,13 @@ func TestWriteMigration_AltersColumnTypeFallsBackToRenameAndAddOnConversionFailu
|
|||||||
}
|
}
|
||||||
|
|
||||||
output := buf.String()
|
output := buf.String()
|
||||||
if !strings.Contains(output, "EXCEPTION WHEN OTHERS THEN") {
|
if !strings.Contains(output, "ALTER COLUMN \"details\" TYPE integer USING \"details\"::integer;") {
|
||||||
t.Fatalf("expected migration to guard the type conversion with an exception handler, got:\n%s", output)
|
t.Fatalf("expected explicit type conversion, got:\n%s", output)
|
||||||
}
|
}
|
||||||
if !strings.Contains(output, "RENAME COLUMN details TO %I") {
|
for _, unsafe := range []string{"EXCEPTION WHEN OTHERS", "renamed_column", "RENAME COLUMN"} {
|
||||||
t.Fatalf("expected migration to rename the old column (derived from the live type) on conversion failure, got:\n%s", output)
|
if strings.Contains(output, unsafe) {
|
||||||
|
t.Fatalf("type conversion must propagate errors without replacing the column (%s):\n%s", unsafe, output)
|
||||||
}
|
}
|
||||||
if !strings.Contains(output, "renamed_column := 'details_' || trim(both '_' from regexp_replace(lower(current_type)") {
|
|
||||||
t.Fatalf("expected migration to derive the renamed column name from the live type, got:\n%s", output)
|
|
||||||
}
|
|
||||||
if !strings.Contains(output, "ADD COLUMN details integer") {
|
|
||||||
t.Fatalf("expected migration to add a fresh column with the new type on conversion failure, got:\n%s", output)
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -214,7 +210,7 @@ func TestWriteMigration_AltersColumnNullabilityWhenNotNullDiffers(t *testing.T)
|
|||||||
}
|
}
|
||||||
|
|
||||||
output := buf.String()
|
output := buf.String()
|
||||||
if !strings.Contains(output, "ALTER COLUMN rid_service_instance_type DROP NOT NULL") {
|
if !strings.Contains(output, "ALTER COLUMN \"rid_service_instance_type\" DROP NOT NULL") {
|
||||||
t.Fatalf("expected migration to drop NOT NULL on existing column, got:\n%s", output)
|
t.Fatalf("expected migration to drop NOT NULL on existing column, got:\n%s", output)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -335,13 +331,13 @@ func TestWriteMigration_UsesStorageTypeForSerialAlterStatements(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
output := buf.String()
|
output := buf.String()
|
||||||
if !strings.Contains(output, "ALTER COLUMN id TYPE bigint") {
|
if !strings.Contains(output, "ALTER COLUMN \"id\" TYPE bigint") {
|
||||||
t.Fatalf("expected serial alter to use bigint storage type, got:\n%s", output)
|
t.Fatalf("expected serial alter to use bigint storage type, got:\n%s", output)
|
||||||
}
|
}
|
||||||
if strings.Contains(output, "ALTER COLUMN id TYPE bigserial;") {
|
if strings.Contains(output, "ALTER COLUMN \"id\" TYPE bigserial;") {
|
||||||
t.Fatalf("did not expect invalid bigserial alter statement, got:\n%s", output)
|
t.Fatalf("did not expect invalid bigserial alter statement, got:\n%s", output)
|
||||||
}
|
}
|
||||||
if !strings.Contains(output, `ALTER COLUMN id TYPE bigint USING id::bigint;`) {
|
if !strings.Contains(output, `ALTER COLUMN "id" TYPE bigint USING "id"::bigint;`) {
|
||||||
t.Fatalf("expected serial alter to include USING cast, got:\n%s", output)
|
t.Fatalf("expected serial alter to include USING cast, got:\n%s", output)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -377,7 +373,7 @@ func TestWriteMigration_ArrayAlterIncludesUsingCast(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
output := buf.String()
|
output := buf.String()
|
||||||
if !strings.Contains(output, `ALTER COLUMN tags TYPE text[] USING tags::text[];`) {
|
if !strings.Contains(output, `ALTER COLUMN "tags" TYPE text[] USING "tags"::text[];`) {
|
||||||
t.Fatalf("expected array alter to include USING cast, got:\n%s", output)
|
t.Fatalf("expected array alter to include USING cast, got:\n%s", output)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -414,7 +410,7 @@ func TestWriteMigration_DoesNotAlterEquivalentNormalizedColumnType(t *testing.T)
|
|||||||
}
|
}
|
||||||
|
|
||||||
output := buf.String()
|
output := buf.String()
|
||||||
if strings.Contains(output, "ALTER COLUMN email TYPE") {
|
if strings.Contains(output, "ALTER COLUMN \"email\" TYPE") {
|
||||||
t.Fatalf("did not expect alter type for equivalent normalized types, got:\n%s", output)
|
t.Fatalf("did not expect alter type for equivalent normalized types, got:\n%s", output)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -497,7 +493,7 @@ func TestWriteMigration_GinIndexOnTextUsesTrigramOperatorClass(t *testing.T) {
|
|||||||
if !strings.Contains(output, "CREATE EXTENSION IF NOT EXISTS pg_trgm;") {
|
if !strings.Contains(output, "CREATE EXTENSION IF NOT EXISTS pg_trgm;") {
|
||||||
t.Fatalf("expected trigram extension for text GIN migration index, got:\n%s", output)
|
t.Fatalf("expected trigram extension for text GIN migration index, got:\n%s", output)
|
||||||
}
|
}
|
||||||
if !strings.Contains(output, "USING gin (title gin_trgm_ops)") {
|
if !strings.Contains(output, "USING gin (\"title\" gin_trgm_ops)") {
|
||||||
t.Fatalf("expected GIN text index to include gin_trgm_ops, got:\n%s", output)
|
t.Fatalf("expected GIN text index to include gin_trgm_ops, got:\n%s", output)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -518,7 +514,7 @@ func TestWriteMigration_GinIndexOnQuotedTextColumnUsesTrigramOperatorClass(t *te
|
|||||||
index := &models.Index{
|
index := &models.Index{
|
||||||
Name: "idx_agent_personas_name_gin",
|
Name: "idx_agent_personas_name_gin",
|
||||||
Type: "gin",
|
Type: "gin",
|
||||||
Columns: []string{`"name"`},
|
Columns: []string{"name"},
|
||||||
}
|
}
|
||||||
table.Indexes[index.Name] = index
|
table.Indexes[index.Name] = index
|
||||||
|
|
||||||
@@ -537,7 +533,7 @@ func TestWriteMigration_GinIndexOnQuotedTextColumnUsesTrigramOperatorClass(t *te
|
|||||||
}
|
}
|
||||||
|
|
||||||
output := buf.String()
|
output := buf.String()
|
||||||
if !strings.Contains(output, "USING gin (name gin_trgm_ops)") {
|
if !strings.Contains(output, "USING gin (\"name\" gin_trgm_ops)") {
|
||||||
t.Fatalf("expected quoted text column GIN index to include gin_trgm_ops, got:\n%s", output)
|
t.Fatalf("expected quoted text column GIN index to include gin_trgm_ops, got:\n%s", output)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -577,7 +573,7 @@ func TestWriteMigration_GinIndexOnTextArrayDoesNotUseTrigramOperatorClass(t *tes
|
|||||||
}
|
}
|
||||||
|
|
||||||
output := buf.String()
|
output := buf.String()
|
||||||
if !strings.Contains(output, "USING gin (tags array_ops)") {
|
if !strings.Contains(output, "USING gin (\"tags\" array_ops)") {
|
||||||
t.Fatalf("expected GIN array index with array_ops, got:\n%s", output)
|
t.Fatalf("expected GIN array index with array_ops, got:\n%s", output)
|
||||||
}
|
}
|
||||||
if strings.Contains(output, "gin_trgm_ops") {
|
if strings.Contains(output, "gin_trgm_ops") {
|
||||||
@@ -620,7 +616,7 @@ func TestWriteMigration_GinIndexOnJSONBUsesJSONBOperatorClass(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
output := buf.String()
|
output := buf.String()
|
||||||
if !strings.Contains(output, "USING gin (details jsonb_ops)") {
|
if !strings.Contains(output, "USING gin (\"details\" jsonb_ops)") {
|
||||||
t.Fatalf("expected GIN jsonb index to include jsonb_ops, got:\n%s", output)
|
t.Fatalf("expected GIN jsonb index to include jsonb_ops, got:\n%s", output)
|
||||||
}
|
}
|
||||||
if strings.Contains(output, "gin_trgm_ops") {
|
if strings.Contains(output, "gin_trgm_ops") {
|
||||||
@@ -664,7 +660,7 @@ func TestWriteMigration_GinIndexOnJSONBIgnoresIncompatibleTrigramOperatorClass(t
|
|||||||
}
|
}
|
||||||
|
|
||||||
output := buf.String()
|
output := buf.String()
|
||||||
if !strings.Contains(output, "USING gin (details jsonb_ops)") {
|
if !strings.Contains(output, "USING gin (\"details\" jsonb_ops)") {
|
||||||
t.Fatalf("expected incompatible trigram hint on jsonb to fall back to jsonb_ops, got:\n%s", output)
|
t.Fatalf("expected incompatible trigram hint on jsonb to fall back to jsonb_ops, got:\n%s", output)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -725,20 +721,20 @@ func TestWriteMigration_WithAudit(t *testing.T) {
|
|||||||
t.Logf("Generated migration with audit:\n%s", output)
|
t.Logf("Generated migration with audit:\n%s", output)
|
||||||
|
|
||||||
// Verify audit tables
|
// Verify audit tables
|
||||||
if !strings.Contains(output, "CREATE TABLE IF NOT EXISTS public.atevent") {
|
if !strings.Contains(output, "CREATE TABLE IF NOT EXISTS \"public\".atevent") {
|
||||||
t.Error("Migration missing atevent table")
|
t.Error("Migration missing atevent table")
|
||||||
}
|
}
|
||||||
if !strings.Contains(output, "CREATE TABLE IF NOT EXISTS public.atdetail") {
|
if !strings.Contains(output, "CREATE TABLE IF NOT EXISTS \"public\".atdetail") {
|
||||||
t.Error("Migration missing atdetail table")
|
t.Error("Migration missing atdetail table")
|
||||||
}
|
}
|
||||||
|
|
||||||
// Verify audit function
|
// Verify audit function
|
||||||
if !strings.Contains(output, "CREATE OR REPLACE FUNCTION public.tf_audit_users()") {
|
if !strings.Contains(output, "CREATE OR REPLACE FUNCTION \"public\".\"tf_audit_users\"()") {
|
||||||
t.Error("Migration missing audit function")
|
t.Error("Migration missing audit function")
|
||||||
}
|
}
|
||||||
|
|
||||||
// Verify audit trigger
|
// Verify audit trigger
|
||||||
if !strings.Contains(output, "CREATE TRIGGER t_audit_users") {
|
if !strings.Contains(output, "CREATE TRIGGER \"t_audit_users\"") {
|
||||||
t.Error("Migration missing audit trigger")
|
t.Error("Migration missing audit trigger")
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -770,13 +766,13 @@ func TestTemplateExecutor_CreateTable(t *testing.T) {
|
|||||||
|
|
||||||
t.Logf("Generated SQL:\n%s", sql)
|
t.Logf("Generated SQL:\n%s", sql)
|
||||||
|
|
||||||
if !strings.Contains(sql, "CREATE TABLE IF NOT EXISTS public.test_table") {
|
if !strings.Contains(sql, "CREATE TABLE IF NOT EXISTS \"public\".\"test_table\"") {
|
||||||
t.Error("SQL missing CREATE TABLE statement")
|
t.Error("SQL missing CREATE TABLE statement")
|
||||||
}
|
}
|
||||||
if !strings.Contains(sql, "id integer NOT NULL") {
|
if !strings.Contains(sql, "\"id\" integer NOT NULL") {
|
||||||
t.Error("SQL missing id column definition")
|
t.Error("SQL missing id column definition")
|
||||||
}
|
}
|
||||||
if !strings.Contains(sql, "name text DEFAULT 'unknown'") {
|
if !strings.Contains(sql, "\"name\" text DEFAULT 'unknown'") {
|
||||||
t.Error("SQL missing name column definition")
|
t.Error("SQL missing name column definition")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -814,7 +810,7 @@ func TestTemplateExecutor_AuditFunction(t *testing.T) {
|
|||||||
|
|
||||||
t.Logf("Generated SQL:\n%s", sql)
|
t.Logf("Generated SQL:\n%s", sql)
|
||||||
|
|
||||||
if !strings.Contains(sql, "CREATE OR REPLACE FUNCTION public.tf_audit_users()") {
|
if !strings.Contains(sql, "CREATE OR REPLACE FUNCTION \"public\".\"tf_audit_users\"()") {
|
||||||
t.Error("SQL missing function definition")
|
t.Error("SQL missing function definition")
|
||||||
}
|
}
|
||||||
if !strings.Contains(sql, "IF TG_OP = 'INSERT'") {
|
if !strings.Contains(sql, "IF TG_OP = 'INSERT'") {
|
||||||
@@ -987,8 +983,8 @@ func TestWriteMigration_VectorAndPostGISIndexes(t *testing.T) {
|
|||||||
"CREATE EXTENSION IF NOT EXISTS vector;",
|
"CREATE EXTENSION IF NOT EXISTS vector;",
|
||||||
"vector(1536)",
|
"vector(1536)",
|
||||||
"geometry(Point,4326)",
|
"geometry(Point,4326)",
|
||||||
"USING ivfflat (embedding vector_cosine_ops) WITH (lists = 100)",
|
"USING ivfflat (\"embedding\" vector_cosine_ops) WITH (lists = 100)",
|
||||||
"USING gist (location)",
|
"USING gist (\"location\")",
|
||||||
} {
|
} {
|
||||||
if !strings.Contains(output, want) {
|
if !strings.Contains(output, want) {
|
||||||
t.Fatalf("expected migration to contain %q, got:\n%s", want, output)
|
t.Fatalf("expected migration to contain %q, got:\n%s", want, output)
|
||||||
@@ -1185,7 +1181,7 @@ func TestWriteMigration_UniqueConstraintAdoptsSameNamedModelIndex(t *testing.T)
|
|||||||
if got := strings.Contains(output, "UNIQUE USING INDEX"); got != tt.wantAdopt {
|
if got := strings.Contains(output, "UNIQUE USING INDEX"); got != tt.wantAdopt {
|
||||||
t.Errorf("USING INDEX present = %v, want %v; output:\n%s", got, tt.wantAdopt, output)
|
t.Errorf("USING INDEX present = %v, want %v; output:\n%s", got, tt.wantAdopt, output)
|
||||||
}
|
}
|
||||||
if got := strings.Contains(output, "UNIQUE (guid)") || strings.Contains(output, "UNIQUE (\"guid\")"); got == tt.wantAdopt {
|
if got := strings.Contains(output, "UNIQUE (\"guid\")"); got == tt.wantAdopt {
|
||||||
t.Errorf("column-list ADD CONSTRAINT present = %v, want %v; output:\n%s", got, !tt.wantAdopt, output)
|
t.Errorf("column-list ADD CONSTRAINT present = %v, want %v; output:\n%s", got, !tt.wantAdopt, output)
|
||||||
}
|
}
|
||||||
if got := strings.Contains(output, "DROP INDEX"); got != tt.wantDrop {
|
if got := strings.Contains(output, "DROP INDEX"); got != tt.wantDrop {
|
||||||
|
|||||||
@@ -0,0 +1,251 @@
|
|||||||
|
package pgsql
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"context"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"git.warky.dev/wdevs/relspecgo/pkg/models"
|
||||||
|
"git.warky.dev/wdevs/relspecgo/pkg/writers"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestQuotedNamesAndLowercaseOption(t *testing.T) {
|
||||||
|
db := models.InitDatabase("DB\nSELECT 'header';")
|
||||||
|
s := models.InitSchema("MySchema")
|
||||||
|
tbl := models.InitTable("Select", "MySchema")
|
||||||
|
col := models.InitColumn("Key", "Select", "MySchema")
|
||||||
|
col.Type = "text"
|
||||||
|
col.Default = "Keep CASE"
|
||||||
|
tbl.Columns[col.Name] = col
|
||||||
|
s.Tables = append(s.Tables, tbl)
|
||||||
|
db.Schemas = append(db.Schemas, s)
|
||||||
|
for _, lower := range []bool{false, true} {
|
||||||
|
w := NewWriter(&writers.WriterOptions{LowercaseIdentifiers: lower})
|
||||||
|
stmts, err := w.GenerateDatabaseStatements(db)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
out := strings.Join(stmts, "\n")
|
||||||
|
want := `"MySchema"."Select"`
|
||||||
|
if lower {
|
||||||
|
want = `"myschema"."select"`
|
||||||
|
}
|
||||||
|
if !strings.Contains(out, want) || !strings.Contains(out, "'Keep CASE'") || strings.Contains(out, "\nSELECT 'header';") {
|
||||||
|
t.Fatalf("%s", out)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if db.Name != "DB\nSELECT 'header';" || tbl.Name != "Select" {
|
||||||
|
t.Fatal("input changed")
|
||||||
|
}
|
||||||
|
path := filepath.Join(t.TempDir(), "out.sql")
|
||||||
|
if err := os.WriteFile(path, []byte("keep"), 0o600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
col.Name = "bad\x00name"
|
||||||
|
if err := NewWriter(&writers.WriterOptions{OutputPath: path}).WriteDatabase(db); err == nil {
|
||||||
|
t.Fatal("expected NUL rejection")
|
||||||
|
}
|
||||||
|
data, _ := os.ReadFile(path)
|
||||||
|
if string(data) != "keep" {
|
||||||
|
t.Fatal("output changed before validation")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAllBlockTemplatesAvoidDelimiterCollisions(t *testing.T) {
|
||||||
|
te, err := NewTemplateExecutor(false)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
name := "名'\"\\\n$$ $body$ $relspec_0$ $relspec_1$"
|
||||||
|
cases := []struct {
|
||||||
|
name string
|
||||||
|
data any
|
||||||
|
}{
|
||||||
|
{"alter_column_default_with_check.tmpl", AlterColumnDefaultWithCheckData{SchemaName: name, TableName: name, ColumnName: name, SetDefault: true, DefaultValue: quote(name)}},
|
||||||
|
{"alter_column_nullability_with_check.tmpl", AlterColumnNullabilityWithCheckData{SchemaName: name, TableName: name, ColumnName: name}},
|
||||||
|
{"create_check_constraint.tmpl", CreateCheckConstraintData{SchemaName: name, TableName: name, ConstraintName: name, Expression: "true"}},
|
||||||
|
{"create_foreign_key_with_check.tmpl", CreateForeignKeyWithCheckData{SchemaName: name, TableName: name, ConstraintName: name}},
|
||||||
|
{"create_primary_key.tmpl", CreatePrimaryKeyData{SchemaName: name, TableName: name, ConstraintName: name}},
|
||||||
|
{"create_primary_key_with_autogen_check.tmpl", CreatePrimaryKeyWithAutoGenCheckData{SchemaName: name, TableName: name, ConstraintName: name}},
|
||||||
|
{"create_unique_constraint.tmpl", CreateUniqueConstraintData{SchemaName: name, TableName: name, ConstraintName: name}},
|
||||||
|
{"set_sequence_value.tmpl", SetSequenceValueData{SchemaName: name, TableName: name, SequenceName: name, ColumnName: name}},
|
||||||
|
{"audit_function.tmpl", AuditFunctionData{SchemaName: name, FunctionName: name, TableName: name, PrimaryKey: name, AuditSchema: name}},
|
||||||
|
{"audit_trigger.tmpl", AuditTriggerData{SchemaName: name, TableName: name, TriggerName: name, FunctionName: name}},
|
||||||
|
}
|
||||||
|
for _, tc := range cases {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
out, err := te.execute(tc.name, tc.data)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if strings.Count(out, "$relspec_2$") != 2 || !strings.Contains(out, quoteIdent(name)) || !strings.Contains(out, quote(name)) {
|
||||||
|
t.Fatalf("%s", out)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestLive_AdversarialNamesAndAudit(t *testing.T) {
|
||||||
|
connString := liveWriterConn(t)
|
||||||
|
base, conn := liveWriterSchema(t, connString)
|
||||||
|
ctx := context.Background()
|
||||||
|
schemaName := base + "'\"\\\n名$relspec_0$"
|
||||||
|
tableName := "select"
|
||||||
|
key := "Key'\"\\\n$relspec_0$"
|
||||||
|
valueName := "名'\"\\\n$body$"
|
||||||
|
comment := "Comment ' \\ $$ $relspec_0$\nUnicode 名"
|
||||||
|
t.Cleanup(func() { _, _ = conn.Exec(ctx, "DROP SCHEMA IF EXISTS "+quoteIdent(schemaName)+" CASCADE") })
|
||||||
|
db := models.InitDatabase("test\nSELECT invalid_header;")
|
||||||
|
s := models.InitSchema(schemaName)
|
||||||
|
tbl := models.InitTable(tableName, schemaName)
|
||||||
|
id := models.InitColumn(key, tableName, schemaName)
|
||||||
|
id.Type = "integer"
|
||||||
|
id.IsPrimaryKey = true
|
||||||
|
id.NotNull = true
|
||||||
|
tbl.Columns[key] = id
|
||||||
|
value := models.InitColumn(valueName, tableName, schemaName)
|
||||||
|
value.Type = "text"
|
||||||
|
value.Default = quote(comment)
|
||||||
|
value.Description = comment
|
||||||
|
value.Comment = comment
|
||||||
|
tbl.Columns[valueName] = value
|
||||||
|
tbl.Description = comment
|
||||||
|
tbl.Comment = comment
|
||||||
|
tbl.Constraints["unique"] = &models.Constraint{Name: "unique'\"\\\n$relspec_0$", Type: models.UniqueConstraint, Columns: []string{valueName}}
|
||||||
|
tbl.Constraints["check"] = &models.Constraint{Name: "check'\"\\\n$relspec_0$", Type: models.CheckConstraint, Expression: quoteIdent(key) + ">0"}
|
||||||
|
tbl.Indexes["index"] = &models.Index{Name: "index'\"\\\n$relspec_0$", Type: "btree", Columns: []string{valueName}}
|
||||||
|
child := models.InitTable("from", schemaName)
|
||||||
|
child.Columns[key] = &models.Column{Name: key, Type: "integer"}
|
||||||
|
child.Constraints["fk"] = &models.Constraint{Name: "foreign'\"\\\n$relspec_0$", Type: models.ForeignKeyConstraint, Columns: []string{key}, ReferencedSchema: schemaName, ReferencedTable: tableName, ReferencedColumns: []string{key}}
|
||||||
|
s.Tables = []*models.Table{tbl, child}
|
||||||
|
db.Schemas = []*models.Schema{s}
|
||||||
|
w := NewWriter(&writers.WriterOptions{})
|
||||||
|
var output bytes.Buffer
|
||||||
|
w.writer = &output
|
||||||
|
if err := w.WriteDatabase(db); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, err := conn.Exec(ctx, output.String()); err != nil {
|
||||||
|
t.Fatalf("file SQL: %v\n%s", err, output.String())
|
||||||
|
}
|
||||||
|
if _, err := conn.Exec(ctx, "SET standard_conforming_strings=off"); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
statements, err := w.GenerateDatabaseStatements(db)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
for _, stmt := range statements {
|
||||||
|
if _, err := conn.Exec(ctx, stmt); err != nil {
|
||||||
|
t.Fatalf("statement: %v\n%s", err, stmt)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
target := w.qualTable(schemaName, tableName)
|
||||||
|
if _, err := conn.Exec(ctx, "INSERT INTO "+target+" ("+quoteIdent(key)+") VALUES (1)"); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
var got string
|
||||||
|
if err := conn.QueryRow(ctx, "SELECT "+quoteIdent(valueName)+" FROM "+target).Scan(&got); err != nil || got != comment {
|
||||||
|
t.Fatalf("default: %q %v", got, err)
|
||||||
|
}
|
||||||
|
if err := conn.QueryRow(ctx, `SELECT col_description(c.oid,a.attnum) FROM pg_class c JOIN pg_namespace n ON n.oid=c.relnamespace JOIN pg_attribute a ON a.attrelid=c.oid WHERE n.nspname=$1 AND c.relname=$2 AND a.attname=$3`, schemaName, tableName, valueName).Scan(&got); err != nil || got != comment {
|
||||||
|
t.Fatalf("comment: %q %v", got, err)
|
||||||
|
}
|
||||||
|
// Migration templates must preserve the same identifier and literal boundaries.
|
||||||
|
current := models.InitDatabase("current")
|
||||||
|
currentSchema := models.InitSchema(schemaName)
|
||||||
|
currentTable := models.InitTable(tableName, schemaName)
|
||||||
|
currentValue := *value
|
||||||
|
currentValue.Type = "text"
|
||||||
|
currentTable.Columns[valueName] = ¤tValue
|
||||||
|
currentID := *id
|
||||||
|
currentTable.Columns[key] = ¤tID
|
||||||
|
currentTable.Constraints = tbl.Constraints
|
||||||
|
currentTable.Indexes = tbl.Indexes
|
||||||
|
currentSchema.Tables = []*models.Table{currentTable, child}
|
||||||
|
current.Schemas = []*models.Schema{currentSchema}
|
||||||
|
value.Type = "varchar"
|
||||||
|
value.Length = 200
|
||||||
|
migration, err := NewMigrationWriter(&writers.WriterOptions{})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
var migrationOutput bytes.Buffer
|
||||||
|
migration.writer = &migrationOutput
|
||||||
|
if err := migration.WriteMigration(db, current); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, err := conn.Exec(ctx, migrationOutput.String()); err != nil {
|
||||||
|
t.Fatalf("migration: %v\n%s", err, migrationOutput.String())
|
||||||
|
}
|
||||||
|
if err := conn.QueryRow(ctx, "SELECT "+quoteIdent(valueName)+" FROM "+target+" /* after migration */").Scan(&got); err != nil || got != comment {
|
||||||
|
t.Fatalf("migration value: %q %v", got, err)
|
||||||
|
}
|
||||||
|
te := w.executor
|
||||||
|
sequenceName := "Seq'\"\\\n$relspec_0$"
|
||||||
|
sequenceSQL, err := te.ExecuteCreateSequence(CreateSequenceData{SchemaName: schemaName, SequenceName: sequenceName, Increment: 1, MinValue: 1, MaxValue: 1000, StartValue: 1, CacheSize: 1})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, err := conn.Exec(ctx, sequenceSQL); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
setvalSQL, err := te.ExecuteSetSequenceValue(SetSequenceValueData{SchemaName: schemaName, TableName: tableName, SequenceName: sequenceName, ColumnName: key})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, err := conn.Exec(ctx, setvalSQL); err != nil {
|
||||||
|
t.Fatalf("sequence: %v\n%s", err, setvalSQL)
|
||||||
|
}
|
||||||
|
var next int
|
||||||
|
if err := conn.QueryRow(ctx, "SELECT nextval($1::regclass)", w.qualTable(schemaName, sequenceName)).Scan(&next); err != nil || next != 2 {
|
||||||
|
t.Fatalf("sequence value: %d %v", next, err)
|
||||||
|
}
|
||||||
|
autoTable := w.qualTable(schemaName, "auto")
|
||||||
|
if _, err := conn.Exec(ctx, "CREATE TABLE "+autoTable+" (old integer PRIMARY KEY, new integer)"); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
pkSQL, err := te.ExecuteCreatePrimaryKeyWithAutoGenCheck(CreatePrimaryKeyWithAutoGenCheckData{SchemaName: schemaName, TableName: "auto", ConstraintName: "new'key", AutoGenNames: formatStringList([]string{"auto_pkey"}), Columns: quotedIdentifiers([]string{"new"}), ColumnNames: formatStringList([]string{"new"})})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, err := conn.Exec(ctx, pkSQL); err != nil {
|
||||||
|
t.Fatalf("primary key: %v\n%s", err, pkSQL)
|
||||||
|
}
|
||||||
|
|
||||||
|
auditTables, err := te.ExecuteAuditTables(AuditTablesData{AuditSchema: schemaName})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, err := conn.Exec(ctx, auditTables); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
config := &TableAuditConfig{AuditInsert: true, AuditUpdate: true, AuditDelete: true, TablePrefix: comment}
|
||||||
|
functionData := BuildAuditFunctionData(schemaName, tbl, id, config, schemaName, "current_user")
|
||||||
|
function, err := te.ExecuteAuditFunction(functionData)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, err := conn.Exec(ctx, function); err != nil {
|
||||||
|
t.Fatalf("audit function: %v\n%s", err, function)
|
||||||
|
}
|
||||||
|
trigger, err := te.ExecuteAuditTrigger(AuditTriggerData{SchemaName: schemaName, TableName: tableName, FunctionName: functionData.FunctionName, TriggerName: "trigger'\"\\\n$relspec_0$", Events: "INSERT OR UPDATE OR DELETE"})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, err := conn.Exec(ctx, trigger); err != nil {
|
||||||
|
t.Fatalf("audit trigger: %v\n%s", err, trigger)
|
||||||
|
}
|
||||||
|
for _, sql := range []string{"INSERT INTO " + target + " VALUES (2, 'second')", "UPDATE " + target + " SET " + quoteIdent(valueName) + "='third' WHERE " + quoteIdent(key) + "=2", "DELETE FROM " + target + " WHERE " + quoteIdent(key) + "=2"} {
|
||||||
|
if _, err := conn.Exec(ctx, sql); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
var count int
|
||||||
|
if err := conn.QueryRow(ctx, "SELECT count(*) FROM "+w.qualTable(schemaName, "atevent")+" WHERE tableprefix=$1", comment).Scan(&count); err != nil || count != 3 {
|
||||||
|
t.Fatalf("audit events: %d %v", count, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -110,7 +110,7 @@ func TestDiffStatements_ExistingTableNewSequenceIsSetPastData(t *testing.T) {
|
|||||||
t.Fatalf("diffStatements failed: %v", err)
|
t.Fatalf("diffStatements failed: %v", err)
|
||||||
}
|
}
|
||||||
out := diffJoin(stmts)
|
out := diffJoin(stmts)
|
||||||
seqIdx := strings.Index(out, "CREATE SEQUENCE IF NOT EXISTS identity.login_event_id_login_event_seq")
|
seqIdx := strings.Index(out, "CREATE SEQUENCE IF NOT EXISTS \"identity\".\"login_event_id_login_event_seq\"")
|
||||||
setvalIdx := strings.Index(out, "setval(")
|
setvalIdx := strings.Index(out, "setval(")
|
||||||
if seqIdx < 0 || setvalIdx < 0 || seqIdx > setvalIdx {
|
if seqIdx < 0 || setvalIdx < 0 || seqIdx > setvalIdx {
|
||||||
t.Fatalf("expected sequence creation before setval, got:\n%s", out)
|
t.Fatalf("expected sequence creation before setval, got:\n%s", out)
|
||||||
|
|||||||
@@ -2,6 +2,7 @@ package pgsql
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"strings"
|
"strings"
|
||||||
@@ -130,7 +131,7 @@ func TestFirstBareIdentAndHelpers(t *testing.T) {
|
|||||||
"abc,def": "abc",
|
"abc,def": "abc",
|
||||||
"abc;": "abc",
|
"abc;": "abc",
|
||||||
"\n abc\tdef": "abc",
|
"\n abc\tdef": "abc",
|
||||||
`"a b" c`: `"a`,
|
`"a b" c`: `"a b"`,
|
||||||
" tbl (x int)": "tbl",
|
" tbl (x int)": "tbl",
|
||||||
}
|
}
|
||||||
for in, want := range bare {
|
for in, want := range bare {
|
||||||
@@ -237,8 +238,9 @@ func TestWriteAndFinishReport(t *testing.T) {
|
|||||||
options: &writers.WriterOptions{Metadata: map[string]interface{}{"report_path": path}},
|
options: &writers.WriterOptions{Metadata: map[string]interface{}{"report_path": path}},
|
||||||
executionReport: report,
|
executionReport: report,
|
||||||
}
|
}
|
||||||
if err := w.finishReport(); err != nil {
|
var failure *ExecutionFailure
|
||||||
t.Fatalf("finishReport: %v", err)
|
if err := w.finishReport(); !errors.As(err, &failure) || failure.Report != report {
|
||||||
|
t.Fatalf("expected structured execution failure, got: %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
data, err := os.ReadFile(path)
|
data, err := os.ReadFile(path)
|
||||||
@@ -270,10 +272,15 @@ func TestWriteReportBadPath(t *testing.T) {
|
|||||||
if err := w.writeReport(filepath.Join(t.TempDir(), "missing", "r.json")); err == nil {
|
if err := w.writeReport(filepath.Join(t.TempDir(), "missing", "r.json")); err == nil {
|
||||||
t.Error("expected error for unwritable path")
|
t.Error("expected error for unwritable path")
|
||||||
}
|
}
|
||||||
// finishReport must swallow the report error.
|
// Explicitly requested reports are part of successful execution.
|
||||||
w.options.Metadata = map[string]interface{}{"report_path": filepath.Join(t.TempDir(), "missing", "r.json")}
|
w.options.Metadata = map[string]interface{}{"report_path": filepath.Join(t.TempDir(), "missing", "r.json")}
|
||||||
if err := w.finishReport(); err != nil {
|
if err := w.finishReport(); err == nil {
|
||||||
t.Errorf("finishReport must not fail on report write error: %v", err)
|
t.Error("finishReport must fail on report write error")
|
||||||
|
}
|
||||||
|
w.executionReport.FailedStatements = 1
|
||||||
|
var failure *ExecutionFailure
|
||||||
|
if err := w.finishReport(); !errors.As(err, &failure) || !strings.Contains(err.Error(), "requested execution report") {
|
||||||
|
t.Fatalf("must preserve both execution and report errors: %v", err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -24,6 +24,7 @@ func TemplateFunctions() map[string]interface{} {
|
|||||||
"escape": escape,
|
"escape": escape,
|
||||||
"safe_identifier": safeIdentifier,
|
"safe_identifier": safeIdentifier,
|
||||||
"quote_ident": quoteIdent,
|
"quote_ident": quoteIdent,
|
||||||
|
"comment_text": commentText,
|
||||||
|
|
||||||
// Type conversion
|
// Type conversion
|
||||||
"goTypeToSQL": goTypeToSQL,
|
"goTypeToSQL": goTypeToSQL,
|
||||||
@@ -99,6 +100,9 @@ func indent(spaces int, text string) string {
|
|||||||
|
|
||||||
// quote quotes a string value for SQL (escapes single quotes)
|
// quote quotes a string value for SQL (escapes single quotes)
|
||||||
func quote(s string) string {
|
func quote(s string) string {
|
||||||
|
if strings.Contains(s, "\\") {
|
||||||
|
return sqlLiteral(s)
|
||||||
|
}
|
||||||
return "'" + strings.ReplaceAll(s, "'", "''") + "'"
|
return "'" + strings.ReplaceAll(s, "'", "''") + "'"
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -125,41 +129,15 @@ func safeIdentifier(s string) string {
|
|||||||
return strings.ToLower(safe)
|
return strings.ToLower(safe)
|
||||||
}
|
}
|
||||||
|
|
||||||
// quoteIdent quotes a PostgreSQL identifier if necessary
|
// quoteIdent always quotes an identifier, preserving case and reserved words.
|
||||||
// Identifiers need quoting if they:
|
func quoteIdent(s string) string { return `"` + strings.ReplaceAll(s, `"`, `""`) + `"` }
|
||||||
// - Start with a digit
|
|
||||||
// - Contain special characters
|
|
||||||
// - Are reserved keywords
|
|
||||||
// - Contain uppercase letters (to preserve case)
|
|
||||||
func quoteIdent(s string) string {
|
|
||||||
if s == "" {
|
|
||||||
return `""`
|
|
||||||
}
|
|
||||||
|
|
||||||
// Check if quoting is needed
|
func quotedIdentifiers(names []string) string {
|
||||||
needsQuoting := unicode.IsDigit(rune(s[0]))
|
out := make([]string, len(names))
|
||||||
|
for i, name := range names {
|
||||||
// Starts with digit
|
out[i] = quoteIdent(name)
|
||||||
|
|
||||||
// Contains uppercase letters or special characters
|
|
||||||
for _, r := range s {
|
|
||||||
if unicode.IsUpper(r) {
|
|
||||||
needsQuoting = true
|
|
||||||
break
|
|
||||||
}
|
}
|
||||||
if !unicode.IsLetter(r) && !unicode.IsDigit(r) && r != '_' {
|
return strings.Join(out, ", ")
|
||||||
needsQuoting = true
|
|
||||||
break
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if needsQuoting {
|
|
||||||
// Escape double quotes by doubling them
|
|
||||||
escaped := strings.ReplaceAll(s, `"`, `""`)
|
|
||||||
return `"` + escaped + `"`
|
|
||||||
}
|
|
||||||
|
|
||||||
return s
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Type conversion functions
|
// Type conversion functions
|
||||||
|
|||||||
@@ -106,7 +106,7 @@ func TestQuoteIdent(t *testing.T) {
|
|||||||
input string
|
input string
|
||||||
expected string
|
expected string
|
||||||
}{
|
}{
|
||||||
{"valid_name", "valid_name"},
|
{"valid_name", `"valid_name"`},
|
||||||
{"ValidName", `"ValidName"`},
|
{"ValidName", `"ValidName"`},
|
||||||
{"123column", `"123column"`},
|
{"123column", `"123column"`},
|
||||||
{"215162_fk_constraint", `"215162_fk_constraint"`},
|
{"215162_fk_constraint", `"215162_fk_constraint"`},
|
||||||
@@ -114,8 +114,8 @@ func TestQuoteIdent(t *testing.T) {
|
|||||||
{"user@domain", `"user@domain"`},
|
{"user@domain", `"user@domain"`},
|
||||||
{`"quoted"`, `"""quoted"""`},
|
{`"quoted"`, `"""quoted"""`},
|
||||||
{"", `""`},
|
{"", `""`},
|
||||||
{"lowercase", "lowercase"},
|
{"lowercase", `"lowercase"`},
|
||||||
{"with_underscore", "with_underscore"},
|
{"with_underscore", `"with_underscore"`},
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, tt := range tests {
|
for _, tt := range tests {
|
||||||
|
|||||||
+69
-155
@@ -8,7 +8,6 @@ import (
|
|||||||
"text/template"
|
"text/template"
|
||||||
|
|
||||||
"git.warky.dev/wdevs/relspecgo/pkg/models"
|
"git.warky.dev/wdevs/relspecgo/pkg/models"
|
||||||
"git.warky.dev/wdevs/relspecgo/pkg/writers"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
//go:embed templates/*.tmpl
|
//go:embed templates/*.tmpl
|
||||||
@@ -29,7 +28,7 @@ func NewTemplateExecutor(flattenSchema bool) (*TemplateExecutor, error) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// qual_table returns a quoted, schema-qualified identifier.
|
// qual_table returns a quoted, schema-qualified identifier.
|
||||||
// With flatten=false: "schema"."table" (or unquoted equivalents).
|
// With flatten=false: "schema"."table".
|
||||||
// With flatten=true: "schema_table".
|
// With flatten=true: "schema_table".
|
||||||
funcMap["qual_table"] = func(schema, name string) string {
|
funcMap["qual_table"] = func(schema, name string) string {
|
||||||
if schema == "" {
|
if schema == "" {
|
||||||
@@ -41,20 +40,20 @@ func NewTemplateExecutor(flattenSchema bool) (*TemplateExecutor, error) {
|
|||||||
return quoteIdent(schema) + "." + quoteIdent(name)
|
return quoteIdent(schema) + "." + quoteIdent(name)
|
||||||
}
|
}
|
||||||
|
|
||||||
// qual_table_raw is the same as qual_table but without identifier quoting.
|
funcMap["dollar_delimiter"] = func() string { return "$$" }
|
||||||
funcMap["qual_table_raw"] = func(schema, name string) string {
|
tmpl := template.New("").Funcs(funcMap)
|
||||||
if schema == "" {
|
entries, err := templateFS.ReadDir("templates")
|
||||||
return name
|
|
||||||
}
|
|
||||||
if flattenSchema {
|
|
||||||
return schema + "_" + name
|
|
||||||
}
|
|
||||||
return schema + "." + name
|
|
||||||
}
|
|
||||||
|
|
||||||
tmpl, err := template.New("").Funcs(funcMap).ParseFS(templateFS, "templates/*.tmpl")
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("failed to parse templates: %w", err)
|
return nil, err
|
||||||
|
}
|
||||||
|
for _, entry := range entries {
|
||||||
|
source, err := templateFS.ReadFile("templates/" + entry.Name())
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if _, err := tmpl.New(entry.Name()).Parse(string(source)); err != nil {
|
||||||
|
return nil, fmt.Errorf("parse %s: %w", entry.Name(), err)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
return &TemplateExecutor{
|
return &TemplateExecutor{
|
||||||
@@ -62,6 +61,31 @@ func NewTemplateExecutor(flattenSchema bool) (*TemplateExecutor, error) {
|
|||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// execute chooses a delimiter absent from rendered data, including SQL expressions.
|
||||||
|
// Only explicit template calls emit delimiters; user text is never rewritten.
|
||||||
|
func (te *TemplateExecutor) execute(name string, data any) (string, error) {
|
||||||
|
for n := 0; ; n++ {
|
||||||
|
delimiter := fmt.Sprintf("$relspec_%d$", n)
|
||||||
|
count := 0
|
||||||
|
tmpl, err := te.templates.Clone()
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
tmpl.Funcs(template.FuncMap{"dollar_delimiter": func() string { count++; return delimiter }})
|
||||||
|
var buf bytes.Buffer
|
||||||
|
if err := tmpl.ExecuteTemplate(&buf, name, data); err != nil {
|
||||||
|
return "", fmt.Errorf("execute %s: %w", name, err)
|
||||||
|
}
|
||||||
|
rendered := buf.String()
|
||||||
|
if strings.ContainsRune(rendered, 0) {
|
||||||
|
return "", fmt.Errorf("%s contains a NUL byte", name)
|
||||||
|
}
|
||||||
|
if strings.Count(rendered, delimiter) == count {
|
||||||
|
return rendered, nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// Template data structures
|
// Template data structures
|
||||||
|
|
||||||
// CreateTableData contains data for create table template
|
// CreateTableData contains data for create table template
|
||||||
@@ -99,8 +123,7 @@ type AddColumnData struct {
|
|||||||
|
|
||||||
// AlterColumnTypeWithCheckData contains data for the guarded alter column
|
// AlterColumnTypeWithCheckData contains data for the guarded alter column
|
||||||
// type template, which only alters existing columns whose live type
|
// type template, which only alters existing columns whose live type
|
||||||
// differs from the desired one, and falls back to renaming the old column
|
// differs from the desired one. Conversion errors propagate unchanged.
|
||||||
// and adding a fresh one when the in-place conversion is not possible.
|
|
||||||
type AlterColumnTypeWithCheckData struct {
|
type AlterColumnTypeWithCheckData struct {
|
||||||
SchemaName string
|
SchemaName string
|
||||||
TableName string
|
TableName string
|
||||||
@@ -108,6 +131,7 @@ type AlterColumnTypeWithCheckData struct {
|
|||||||
NewType string
|
NewType string
|
||||||
EquivalentTypes string
|
EquivalentTypes string
|
||||||
UsingExpr string
|
UsingExpr string
|
||||||
|
PreflightPredicate string
|
||||||
}
|
}
|
||||||
|
|
||||||
// AlterColumnDefaultWithCheckData contains data for the guarded alter
|
// AlterColumnDefaultWithCheckData contains data for the guarded alter
|
||||||
@@ -305,227 +329,117 @@ type CreatePrimaryKeyWithAutoGenCheckData struct {
|
|||||||
|
|
||||||
// ExecuteCreateTable executes the create table template
|
// ExecuteCreateTable executes the create table template
|
||||||
func (te *TemplateExecutor) ExecuteCreateTable(data CreateTableData) (string, error) {
|
func (te *TemplateExecutor) ExecuteCreateTable(data CreateTableData) (string, error) {
|
||||||
var buf bytes.Buffer
|
return te.execute("create_table.tmpl", data)
|
||||||
err := te.templates.ExecuteTemplate(&buf, "create_table.tmpl", data)
|
|
||||||
if err != nil {
|
|
||||||
return "", fmt.Errorf("failed to execute create_table template: %w", err)
|
|
||||||
}
|
|
||||||
return buf.String(), nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// ExecuteAddColumn executes the add column template
|
// ExecuteAddColumn executes the add column template
|
||||||
func (te *TemplateExecutor) ExecuteAddColumn(data AddColumnData) (string, error) {
|
func (te *TemplateExecutor) ExecuteAddColumn(data AddColumnData) (string, error) {
|
||||||
var buf bytes.Buffer
|
return te.execute("add_column.tmpl", data)
|
||||||
err := te.templates.ExecuteTemplate(&buf, "add_column.tmpl", data)
|
|
||||||
if err != nil {
|
|
||||||
return "", fmt.Errorf("failed to execute add_column template: %w", err)
|
|
||||||
}
|
|
||||||
return buf.String(), nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// ExecuteAlterColumnTypeWithCheck executes the guarded alter column type
|
// ExecuteAlterColumnTypeWithCheck executes the guarded alter column type
|
||||||
// template shared by the full-schema writer and the diff-based migration
|
// template shared by the full-schema writer and the diff-based migration
|
||||||
// writer.
|
// writer.
|
||||||
func (te *TemplateExecutor) ExecuteAlterColumnTypeWithCheck(data AlterColumnTypeWithCheckData) (string, error) {
|
func (te *TemplateExecutor) ExecuteAlterColumnTypeWithCheck(data AlterColumnTypeWithCheckData) (string, error) {
|
||||||
var buf bytes.Buffer
|
return te.execute("alter_column_type_with_check.tmpl", data)
|
||||||
err := te.templates.ExecuteTemplate(&buf, "alter_column_type_with_check.tmpl", data)
|
|
||||||
if err != nil {
|
|
||||||
return "", fmt.Errorf("failed to execute alter_column_type_with_check template: %w", err)
|
|
||||||
}
|
|
||||||
return buf.String(), nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// ExecuteAlterColumnDefaultWithCheck executes the guarded alter column
|
// ExecuteAlterColumnDefaultWithCheck executes the guarded alter column
|
||||||
// default template shared by the full-schema writer and the diff-based
|
// default template shared by the full-schema writer and the diff-based
|
||||||
// migration writer.
|
// migration writer.
|
||||||
func (te *TemplateExecutor) ExecuteAlterColumnDefaultWithCheck(data AlterColumnDefaultWithCheckData) (string, error) {
|
func (te *TemplateExecutor) ExecuteAlterColumnDefaultWithCheck(data AlterColumnDefaultWithCheckData) (string, error) {
|
||||||
var buf bytes.Buffer
|
return te.execute("alter_column_default_with_check.tmpl", data)
|
||||||
err := te.templates.ExecuteTemplate(&buf, "alter_column_default_with_check.tmpl", data)
|
|
||||||
if err != nil {
|
|
||||||
return "", fmt.Errorf("failed to execute alter_column_default_with_check template: %w", err)
|
|
||||||
}
|
|
||||||
return buf.String(), nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// ExecuteAlterColumnNullabilityWithCheck executes the guarded alter column
|
// ExecuteAlterColumnNullabilityWithCheck executes the guarded alter column
|
||||||
// nullability template.
|
// nullability template.
|
||||||
func (te *TemplateExecutor) ExecuteAlterColumnNullabilityWithCheck(data AlterColumnNullabilityWithCheckData) (string, error) {
|
func (te *TemplateExecutor) ExecuteAlterColumnNullabilityWithCheck(data AlterColumnNullabilityWithCheckData) (string, error) {
|
||||||
var buf bytes.Buffer
|
return te.execute("alter_column_nullability_with_check.tmpl", data)
|
||||||
err := te.templates.ExecuteTemplate(&buf, "alter_column_nullability_with_check.tmpl", data)
|
|
||||||
if err != nil {
|
|
||||||
return "", fmt.Errorf("failed to execute alter_column_nullability_with_check template: %w", err)
|
|
||||||
}
|
|
||||||
return buf.String(), nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// ExecuteCreatePrimaryKey executes the create primary key template
|
// ExecuteCreatePrimaryKey executes the create primary key template
|
||||||
func (te *TemplateExecutor) ExecuteCreatePrimaryKey(data CreatePrimaryKeyData) (string, error) {
|
func (te *TemplateExecutor) ExecuteCreatePrimaryKey(data CreatePrimaryKeyData) (string, error) {
|
||||||
var buf bytes.Buffer
|
return te.execute("create_primary_key.tmpl", data)
|
||||||
err := te.templates.ExecuteTemplate(&buf, "create_primary_key.tmpl", data)
|
|
||||||
if err != nil {
|
|
||||||
return "", fmt.Errorf("failed to execute create_primary_key template: %w", err)
|
|
||||||
}
|
|
||||||
return buf.String(), nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// ExecuteCreateIndex executes the create index template
|
// ExecuteCreateIndex executes the create index template
|
||||||
func (te *TemplateExecutor) ExecuteCreateIndex(data CreateIndexData) (string, error) {
|
func (te *TemplateExecutor) ExecuteCreateIndex(data CreateIndexData) (string, error) {
|
||||||
var buf bytes.Buffer
|
return te.execute("create_index.tmpl", data)
|
||||||
err := te.templates.ExecuteTemplate(&buf, "create_index.tmpl", data)
|
|
||||||
if err != nil {
|
|
||||||
return "", fmt.Errorf("failed to execute create_index template: %w", err)
|
|
||||||
}
|
|
||||||
return buf.String(), nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// ExecuteCreateForeignKey executes the create foreign key template
|
// ExecuteCreateForeignKey executes the create foreign key template
|
||||||
func (te *TemplateExecutor) ExecuteCreateForeignKey(data CreateForeignKeyData) (string, error) {
|
func (te *TemplateExecutor) ExecuteCreateForeignKey(data CreateForeignKeyData) (string, error) {
|
||||||
var buf bytes.Buffer
|
return te.execute("create_foreign_key.tmpl", data)
|
||||||
err := te.templates.ExecuteTemplate(&buf, "create_foreign_key.tmpl", data)
|
|
||||||
if err != nil {
|
|
||||||
return "", fmt.Errorf("failed to execute create_foreign_key template: %w", err)
|
|
||||||
}
|
|
||||||
return buf.String(), nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// ExecuteDropConstraint executes the drop constraint template
|
// ExecuteDropConstraint executes the drop constraint template
|
||||||
func (te *TemplateExecutor) ExecuteDropConstraint(data DropConstraintData) (string, error) {
|
func (te *TemplateExecutor) ExecuteDropConstraint(data DropConstraintData) (string, error) {
|
||||||
var buf bytes.Buffer
|
return te.execute("drop_constraint.tmpl", data)
|
||||||
err := te.templates.ExecuteTemplate(&buf, "drop_constraint.tmpl", data)
|
|
||||||
if err != nil {
|
|
||||||
return "", fmt.Errorf("failed to execute drop_constraint template: %w", err)
|
|
||||||
}
|
|
||||||
return buf.String(), nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// ExecuteDropIndex executes the drop index template
|
// ExecuteDropIndex executes the drop index template
|
||||||
func (te *TemplateExecutor) ExecuteDropIndex(data DropIndexData) (string, error) {
|
func (te *TemplateExecutor) ExecuteDropIndex(data DropIndexData) (string, error) {
|
||||||
var buf bytes.Buffer
|
return te.execute("drop_index.tmpl", data)
|
||||||
err := te.templates.ExecuteTemplate(&buf, "drop_index.tmpl", data)
|
|
||||||
if err != nil {
|
|
||||||
return "", fmt.Errorf("failed to execute drop_index template: %w", err)
|
|
||||||
}
|
|
||||||
return buf.String(), nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// ExecuteCommentTable executes the table comment template
|
// ExecuteCommentTable executes the table comment template
|
||||||
func (te *TemplateExecutor) ExecuteCommentTable(data CommentTableData) (string, error) {
|
func (te *TemplateExecutor) ExecuteCommentTable(data CommentTableData) (string, error) {
|
||||||
var buf bytes.Buffer
|
return te.execute("comment_table.tmpl", data)
|
||||||
err := te.templates.ExecuteTemplate(&buf, "comment_table.tmpl", data)
|
|
||||||
if err != nil {
|
|
||||||
return "", fmt.Errorf("failed to execute comment_table template: %w", err)
|
|
||||||
}
|
|
||||||
return buf.String(), nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// ExecuteCommentColumn executes the column comment template
|
// ExecuteCommentColumn executes the column comment template
|
||||||
func (te *TemplateExecutor) ExecuteCommentColumn(data CommentColumnData) (string, error) {
|
func (te *TemplateExecutor) ExecuteCommentColumn(data CommentColumnData) (string, error) {
|
||||||
var buf bytes.Buffer
|
return te.execute("comment_column.tmpl", data)
|
||||||
err := te.templates.ExecuteTemplate(&buf, "comment_column.tmpl", data)
|
|
||||||
if err != nil {
|
|
||||||
return "", fmt.Errorf("failed to execute comment_column template: %w", err)
|
|
||||||
}
|
|
||||||
return buf.String(), nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// ExecuteAuditTables executes the audit tables template
|
// ExecuteAuditTables executes the audit tables template
|
||||||
func (te *TemplateExecutor) ExecuteAuditTables(data AuditTablesData) (string, error) {
|
func (te *TemplateExecutor) ExecuteAuditTables(data AuditTablesData) (string, error) {
|
||||||
var buf bytes.Buffer
|
return te.execute("audit_tables.tmpl", data)
|
||||||
err := te.templates.ExecuteTemplate(&buf, "audit_tables.tmpl", data)
|
|
||||||
if err != nil {
|
|
||||||
return "", fmt.Errorf("failed to execute audit_tables template: %w", err)
|
|
||||||
}
|
|
||||||
return buf.String(), nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// ExecuteAuditFunction executes the audit function template
|
// ExecuteAuditFunction executes the audit function template
|
||||||
func (te *TemplateExecutor) ExecuteAuditFunction(data AuditFunctionData) (string, error) {
|
func (te *TemplateExecutor) ExecuteAuditFunction(data AuditFunctionData) (string, error) {
|
||||||
var buf bytes.Buffer
|
return te.execute("audit_function.tmpl", data)
|
||||||
err := te.templates.ExecuteTemplate(&buf, "audit_function.tmpl", data)
|
|
||||||
if err != nil {
|
|
||||||
return "", fmt.Errorf("failed to execute audit_function template: %w", err)
|
|
||||||
}
|
|
||||||
return buf.String(), nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// ExecuteAuditTrigger executes the audit trigger template
|
// ExecuteAuditTrigger executes the audit trigger template
|
||||||
func (te *TemplateExecutor) ExecuteAuditTrigger(data AuditTriggerData) (string, error) {
|
func (te *TemplateExecutor) ExecuteAuditTrigger(data AuditTriggerData) (string, error) {
|
||||||
var buf bytes.Buffer
|
return te.execute("audit_trigger.tmpl", data)
|
||||||
err := te.templates.ExecuteTemplate(&buf, "audit_trigger.tmpl", data)
|
|
||||||
if err != nil {
|
|
||||||
return "", fmt.Errorf("failed to execute audit_trigger template: %w", err)
|
|
||||||
}
|
|
||||||
return buf.String(), nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// ExecuteCreateUniqueConstraint executes the create unique constraint template
|
// ExecuteCreateUniqueConstraint executes the create unique constraint template
|
||||||
func (te *TemplateExecutor) ExecuteCreateUniqueConstraint(data CreateUniqueConstraintData) (string, error) {
|
func (te *TemplateExecutor) ExecuteCreateUniqueConstraint(data CreateUniqueConstraintData) (string, error) {
|
||||||
var buf bytes.Buffer
|
return te.execute("create_unique_constraint.tmpl", data)
|
||||||
err := te.templates.ExecuteTemplate(&buf, "create_unique_constraint.tmpl", data)
|
|
||||||
if err != nil {
|
|
||||||
return "", fmt.Errorf("failed to execute create_unique_constraint template: %w", err)
|
|
||||||
}
|
|
||||||
return buf.String(), nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// ExecuteCreateCheckConstraint executes the create check constraint template
|
// ExecuteCreateCheckConstraint executes the create check constraint template
|
||||||
func (te *TemplateExecutor) ExecuteCreateCheckConstraint(data CreateCheckConstraintData) (string, error) {
|
func (te *TemplateExecutor) ExecuteCreateCheckConstraint(data CreateCheckConstraintData) (string, error) {
|
||||||
var buf bytes.Buffer
|
return te.execute("create_check_constraint.tmpl", data)
|
||||||
err := te.templates.ExecuteTemplate(&buf, "create_check_constraint.tmpl", data)
|
|
||||||
if err != nil {
|
|
||||||
return "", fmt.Errorf("failed to execute create_check_constraint template: %w", err)
|
|
||||||
}
|
|
||||||
return buf.String(), nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// ExecuteCreateForeignKeyWithCheck executes the create foreign key with check template
|
// ExecuteCreateForeignKeyWithCheck executes the create foreign key with check template
|
||||||
func (te *TemplateExecutor) ExecuteCreateForeignKeyWithCheck(data CreateForeignKeyWithCheckData) (string, error) {
|
func (te *TemplateExecutor) ExecuteCreateForeignKeyWithCheck(data CreateForeignKeyWithCheckData) (string, error) {
|
||||||
var buf bytes.Buffer
|
return te.execute("create_foreign_key_with_check.tmpl", data)
|
||||||
err := te.templates.ExecuteTemplate(&buf, "create_foreign_key_with_check.tmpl", data)
|
|
||||||
if err != nil {
|
|
||||||
return "", fmt.Errorf("failed to execute create_foreign_key_with_check template: %w", err)
|
|
||||||
}
|
|
||||||
return buf.String(), nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// ExecuteSetSequenceValue executes the set sequence value template
|
// ExecuteSetSequenceValue executes the set sequence value template
|
||||||
func (te *TemplateExecutor) ExecuteSetSequenceValue(data SetSequenceValueData) (string, error) {
|
func (te *TemplateExecutor) ExecuteSetSequenceValue(data SetSequenceValueData) (string, error) {
|
||||||
var buf bytes.Buffer
|
return te.execute("set_sequence_value.tmpl", data)
|
||||||
err := te.templates.ExecuteTemplate(&buf, "set_sequence_value.tmpl", data)
|
|
||||||
if err != nil {
|
|
||||||
return "", fmt.Errorf("failed to execute set_sequence_value template: %w", err)
|
|
||||||
}
|
|
||||||
return buf.String(), nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// ExecuteCreateSequence executes the create sequence template
|
// ExecuteCreateSequence executes the create sequence template
|
||||||
func (te *TemplateExecutor) ExecuteCreateSequence(data CreateSequenceData) (string, error) {
|
func (te *TemplateExecutor) ExecuteCreateSequence(data CreateSequenceData) (string, error) {
|
||||||
var buf bytes.Buffer
|
return te.execute("create_sequence.tmpl", data)
|
||||||
err := te.templates.ExecuteTemplate(&buf, "create_sequence.tmpl", data)
|
|
||||||
if err != nil {
|
|
||||||
return "", fmt.Errorf("failed to execute create_sequence template: %w", err)
|
|
||||||
}
|
|
||||||
return buf.String(), nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// ExecuteAddColumnWithCheck executes the add column with check template
|
// ExecuteAddColumnWithCheck executes the add column with check template
|
||||||
func (te *TemplateExecutor) ExecuteAddColumnWithCheck(data AddColumnWithCheckData) (string, error) {
|
func (te *TemplateExecutor) ExecuteAddColumnWithCheck(data AddColumnWithCheckData) (string, error) {
|
||||||
var buf bytes.Buffer
|
return te.execute("add_column_with_check.tmpl", data)
|
||||||
err := te.templates.ExecuteTemplate(&buf, "add_column_with_check.tmpl", data)
|
|
||||||
if err != nil {
|
|
||||||
return "", fmt.Errorf("failed to execute add_column_with_check template: %w", err)
|
|
||||||
}
|
|
||||||
return buf.String(), nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// ExecuteCreatePrimaryKeyWithAutoGenCheck executes the create primary key with auto-generated key check template
|
// ExecuteCreatePrimaryKeyWithAutoGenCheck executes the create primary key with auto-generated key check template
|
||||||
func (te *TemplateExecutor) ExecuteCreatePrimaryKeyWithAutoGenCheck(data CreatePrimaryKeyWithAutoGenCheckData) (string, error) {
|
func (te *TemplateExecutor) ExecuteCreatePrimaryKeyWithAutoGenCheck(data CreatePrimaryKeyWithAutoGenCheckData) (string, error) {
|
||||||
var buf bytes.Buffer
|
return te.execute("create_primary_key_with_autogen_check.tmpl", data)
|
||||||
err := te.templates.ExecuteTemplate(&buf, "create_primary_key_with_autogen_check.tmpl", data)
|
|
||||||
if err != nil {
|
|
||||||
return "", fmt.Errorf("failed to execute create_primary_key_with_autogen_check template: %w", err)
|
|
||||||
}
|
|
||||||
return buf.String(), nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Helper functions to build template data from models
|
// Helper functions to build template data from models
|
||||||
@@ -558,7 +472,7 @@ func BuildCreateTableData(schemaName string, table *models.Table) CreateTableDat
|
|||||||
colData.Generated, colData.Identity = columnGenerationClauses(col)
|
colData.Generated, colData.Identity = columnGenerationClauses(col)
|
||||||
if col.Default != nil && colData.Generated == "" && colData.Identity == "" {
|
if col.Default != nil && colData.Generated == "" && colData.Identity == "" {
|
||||||
if value, ok := col.Default.(string); ok {
|
if value, ok := col.Default.(string); ok {
|
||||||
colData.Default = writers.QuoteDefaultValue(stripBackticks(value), col.Type)
|
colData.Default = quoteDefaultValue(stripBackticks(value), col.Type)
|
||||||
} else {
|
} else {
|
||||||
colData.Default = fmt.Sprintf("%v", col.Default)
|
colData.Default = fmt.Sprintf("%v", col.Default)
|
||||||
}
|
}
|
||||||
@@ -609,7 +523,7 @@ func BuildAuditFunctionData(
|
|||||||
updateComparisons := make([]string, 0)
|
updateComparisons := make([]string, 0)
|
||||||
for _, col := range auditedColumns {
|
for _, col := range auditedColumns {
|
||||||
updateComparisons = append(updateComparisons,
|
updateComparisons = append(updateComparisons,
|
||||||
fmt.Sprintf("old.%s IS DISTINCT FROM new.%s", col.Name, col.Name))
|
fmt.Sprintf("old.%s IS DISTINCT FROM new.%s", quoteIdent(col.Name), quoteIdent(col.Name)))
|
||||||
}
|
}
|
||||||
updateCondition := strings.Join(updateComparisons, " OR ")
|
updateCondition := strings.Join(updateComparisons, " OR ")
|
||||||
|
|
||||||
@@ -624,8 +538,8 @@ func BuildAuditFunctionData(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
oldValue := fmt.Sprintf("old.%s::text", col.Name)
|
oldValue := fmt.Sprintf("old.%s::text", quoteIdent(col.Name))
|
||||||
newValue := fmt.Sprintf("new.%s::text", col.Name)
|
newValue := fmt.Sprintf("new.%s::text", quoteIdent(col.Name))
|
||||||
|
|
||||||
if isEncrypted {
|
if isEncrypted {
|
||||||
oldValue = "'****************'"
|
oldValue = "'****************'"
|
||||||
@@ -650,7 +564,7 @@ func BuildAuditFunctionData(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
oldValue := fmt.Sprintf("old.%s::text", col.Name)
|
oldValue := fmt.Sprintf("old.%s::text", quoteIdent(col.Name))
|
||||||
if isEncrypted {
|
if isEncrypted {
|
||||||
oldValue = "'****************'"
|
oldValue = "'****************'"
|
||||||
}
|
}
|
||||||
@@ -663,7 +577,7 @@ func BuildAuditFunctionData(
|
|||||||
|
|
||||||
tablePrefix := "NULL"
|
tablePrefix := "NULL"
|
||||||
if config.TablePrefix != "" {
|
if config.TablePrefix != "" {
|
||||||
tablePrefix = fmt.Sprintf("'%s'", config.TablePrefix)
|
tablePrefix = quote(config.TablePrefix)
|
||||||
}
|
}
|
||||||
|
|
||||||
return AuditFunctionData{
|
return AuditFunctionData{
|
||||||
|
|||||||
@@ -1,12 +1 @@
|
|||||||
DO $$
|
ALTER TABLE {{qual_table .SchemaName .TableName}} ADD COLUMN IF NOT EXISTS {{.ColumnDefinition}};
|
||||||
BEGIN
|
|
||||||
IF NOT EXISTS (
|
|
||||||
SELECT 1 FROM information_schema.columns
|
|
||||||
WHERE table_schema = '{{.SchemaName}}'
|
|
||||||
AND table_name = '{{.TableName}}'
|
|
||||||
AND column_name = '{{.ColumnName}}'
|
|
||||||
) THEN
|
|
||||||
ALTER TABLE {{qual_table .SchemaName .TableName}} ADD COLUMN {{.ColumnDefinition}};
|
|
||||||
END IF;
|
|
||||||
END;
|
|
||||||
$$;
|
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
DO $$
|
DO {{dollar_delimiter}}
|
||||||
DECLARE
|
DECLARE
|
||||||
current_default text;
|
current_default text;
|
||||||
BEGIN
|
BEGIN
|
||||||
@@ -8,9 +8,9 @@ BEGIN
|
|||||||
JOIN pg_class t ON t.oid = a.attrelid
|
JOIN pg_class t ON t.oid = a.attrelid
|
||||||
JOIN pg_namespace n ON n.oid = t.relnamespace
|
JOIN pg_namespace n ON n.oid = t.relnamespace
|
||||||
LEFT JOIN pg_attrdef d ON d.adrelid = a.attrelid AND d.adnum = a.attnum
|
LEFT JOIN pg_attrdef d ON d.adrelid = a.attrelid AND d.adnum = a.attnum
|
||||||
WHERE n.nspname = '{{.SchemaName}}'
|
WHERE n.nspname = {{quote .SchemaName}}
|
||||||
AND t.relname = '{{.TableName}}'
|
AND t.relname = {{quote .TableName}}
|
||||||
AND a.attname = '{{.ColumnName}}'
|
AND a.attname = {{quote .ColumnName}}
|
||||||
AND a.attnum > 0
|
AND a.attnum > 0
|
||||||
AND NOT a.attisdropped;
|
AND NOT a.attisdropped;
|
||||||
|
|
||||||
@@ -26,4 +26,4 @@ BEGIN
|
|||||||
END IF;
|
END IF;
|
||||||
{{- end }}
|
{{- end }}
|
||||||
END;
|
END;
|
||||||
$$;
|
{{dollar_delimiter}};
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
DO $$
|
DO {{dollar_delimiter}}
|
||||||
DECLARE
|
DECLARE
|
||||||
current_not_null boolean;
|
current_not_null boolean;
|
||||||
BEGIN
|
BEGIN
|
||||||
@@ -7,9 +7,9 @@ BEGIN
|
|||||||
FROM pg_attribute a
|
FROM pg_attribute a
|
||||||
JOIN pg_class t ON t.oid = a.attrelid
|
JOIN pg_class t ON t.oid = a.attrelid
|
||||||
JOIN pg_namespace n ON n.oid = t.relnamespace
|
JOIN pg_namespace n ON n.oid = t.relnamespace
|
||||||
WHERE n.nspname = '{{.SchemaName}}'
|
WHERE n.nspname = {{quote .SchemaName}}
|
||||||
AND t.relname = '{{.TableName}}'
|
AND t.relname = {{quote .TableName}}
|
||||||
AND a.attname = '{{.ColumnName}}'
|
AND a.attname = {{quote .ColumnName}}
|
||||||
AND a.attnum > 0
|
AND a.attnum > 0
|
||||||
AND NOT a.attisdropped;
|
AND NOT a.attisdropped;
|
||||||
|
|
||||||
@@ -23,4 +23,4 @@ BEGIN
|
|||||||
{{- end }}
|
{{- end }}
|
||||||
END IF;
|
END IF;
|
||||||
END;
|
END;
|
||||||
$$;
|
{{dollar_delimiter}};
|
||||||
|
|||||||
@@ -1,30 +1,22 @@
|
|||||||
DO $$
|
DO {{dollar_delimiter}}
|
||||||
DECLARE
|
DECLARE
|
||||||
current_type text;
|
current_type text;
|
||||||
renamed_column text;
|
|
||||||
BEGIN
|
BEGIN
|
||||||
SELECT pg_catalog.format_type(a.atttypid, a.atttypmod)
|
SELECT pg_catalog.format_type(a.atttypid, a.atttypmod)
|
||||||
INTO current_type
|
INTO current_type
|
||||||
FROM pg_attribute a
|
FROM pg_attribute a
|
||||||
JOIN pg_class t ON t.oid = a.attrelid
|
JOIN pg_class t ON t.oid = a.attrelid
|
||||||
JOIN pg_namespace n ON n.oid = t.relnamespace
|
JOIN pg_namespace n ON n.oid = t.relnamespace
|
||||||
WHERE n.nspname = '{{.SchemaName}}'
|
WHERE n.nspname = {{quote .SchemaName}}
|
||||||
AND t.relname = '{{.TableName}}'
|
AND t.relname = {{quote .TableName}}
|
||||||
AND a.attname = '{{.ColumnName}}'
|
AND a.attname = {{quote .ColumnName}}
|
||||||
AND a.attnum > 0
|
AND a.attnum > 0
|
||||||
AND NOT a.attisdropped;
|
AND NOT a.attisdropped;
|
||||||
|
|
||||||
IF current_type IS NOT NULL
|
IF current_type IS NOT NULL
|
||||||
AND current_type <> ALL(ARRAY[{{.EquivalentTypes}}]) THEN
|
AND current_type <> ALL(ARRAY[{{.EquivalentTypes}}]) THEN
|
||||||
BEGIN
|
|
||||||
ALTER TABLE {{qual_table .SchemaName .TableName}}
|
ALTER TABLE {{qual_table .SchemaName .TableName}}
|
||||||
ALTER COLUMN {{quote_ident .ColumnName}} TYPE {{.NewType}}{{if .UsingExpr}} USING {{.UsingExpr}}{{end}};
|
ALTER COLUMN {{quote_ident .ColumnName}} TYPE {{.NewType}}{{if .UsingExpr}} USING {{.UsingExpr}}{{end}};
|
||||||
EXCEPTION WHEN OTHERS THEN
|
|
||||||
renamed_column := '{{.ColumnName}}_' || trim(both '_' from regexp_replace(lower(current_type), '[^a-z0-9]+', '_', 'g'));
|
|
||||||
EXECUTE format('ALTER TABLE {{qual_table .SchemaName .TableName}} RENAME COLUMN {{quote_ident .ColumnName}} TO %I', renamed_column);
|
|
||||||
ALTER TABLE {{qual_table .SchemaName .TableName}}
|
|
||||||
ADD COLUMN {{quote_ident .ColumnName}} {{.NewType}};
|
|
||||||
END;
|
|
||||||
END IF;
|
END IF;
|
||||||
END;
|
END;
|
||||||
$$;
|
{{dollar_delimiter}};
|
||||||
|
|||||||
@@ -1,8 +1,8 @@
|
|||||||
CREATE OR REPLACE FUNCTION {{qual_table_raw .SchemaName .FunctionName}}()
|
CREATE OR REPLACE FUNCTION {{qual_table .SchemaName .FunctionName}}()
|
||||||
RETURNS trigger AS
|
RETURNS trigger AS
|
||||||
$body$
|
{{dollar_delimiter}}
|
||||||
DECLARE
|
DECLARE
|
||||||
m_funcname text = '{{.FunctionName}}';
|
m_funcname text = {{quote .FunctionName}};
|
||||||
m_user text;
|
m_user text;
|
||||||
m_atevent integer;
|
m_atevent integer;
|
||||||
BEGIN
|
BEGIN
|
||||||
@@ -23,8 +23,8 @@ BEGIN
|
|||||||
{{- if .AuditInsert}}
|
{{- if .AuditInsert}}
|
||||||
IF TG_OP = 'INSERT' THEN
|
IF TG_OP = 'INSERT' THEN
|
||||||
-- Record INSERT
|
-- Record INSERT
|
||||||
INSERT INTO {{.AuditSchema}}.atevent (tablename, tableprefix, rid_parent, changeuser, changedate, changetime, actionx)
|
INSERT INTO {{quote_ident .AuditSchema}}.atevent (tablename, tableprefix, rid_parent, changeuser, changedate, changetime, actionx)
|
||||||
VALUES ('{{.TableName}}', {{.TablePrefix}}, new.{{.PrimaryKey}}, m_user, CURRENT_DATE, CURRENT_TIME, 1)
|
VALUES ({{quote .TableName}}, {{.TablePrefix}}, new.{{quote_ident .PrimaryKey}}, m_user, CURRENT_DATE, CURRENT_TIME, 1)
|
||||||
RETURNING rid_atevent INTO m_atevent;
|
RETURNING rid_atevent INTO m_atevent;
|
||||||
{{- end}}
|
{{- end}}
|
||||||
|
|
||||||
@@ -32,15 +32,15 @@ BEGIN
|
|||||||
ELSIF TG_OP = 'UPDATE' THEN
|
ELSIF TG_OP = 'UPDATE' THEN
|
||||||
-- Check if any audited columns changed
|
-- Check if any audited columns changed
|
||||||
IF ({{.UpdateCondition}}) THEN
|
IF ({{.UpdateCondition}}) THEN
|
||||||
INSERT INTO {{.AuditSchema}}.atevent (tablename, tableprefix, rid_parent, changeuser, changedate, changetime, actionx)
|
INSERT INTO {{quote_ident .AuditSchema}}.atevent (tablename, tableprefix, rid_parent, changeuser, changedate, changetime, actionx)
|
||||||
VALUES ('{{.TableName}}', {{.TablePrefix}}, new.{{.PrimaryKey}}, m_user, CURRENT_DATE, CURRENT_TIME, 2)
|
VALUES ({{quote .TableName}}, {{.TablePrefix}}, new.{{quote_ident .PrimaryKey}}, m_user, CURRENT_DATE, CURRENT_TIME, 2)
|
||||||
RETURNING rid_atevent INTO m_atevent;
|
RETURNING rid_atevent INTO m_atevent;
|
||||||
|
|
||||||
-- Record column changes
|
-- Record column changes
|
||||||
{{- range .UpdateColumns}}
|
{{- range .UpdateColumns}}
|
||||||
IF (old.{{.Name}} IS DISTINCT FROM new.{{.Name}}) THEN
|
IF (old.{{quote_ident .Name}} IS DISTINCT FROM new.{{quote_ident .Name}}) THEN
|
||||||
INSERT INTO {{$.AuditSchema}}.atdetail(rid_atevent, datacolumn, changedfrom, changedto)
|
INSERT INTO {{quote_ident $.AuditSchema}}.atdetail(rid_atevent, datacolumn, changedfrom, changedto)
|
||||||
VALUES (m_atevent, '{{.Name}}', substr({{.OldValue}}, 1, 1000), substr({{.NewValue}}, 1, 1000));
|
VALUES (m_atevent, {{quote .Name}}, substr({{.OldValue}}, 1, 1000), substr({{.NewValue}}, 1, 1000));
|
||||||
END IF;
|
END IF;
|
||||||
{{- end}}
|
{{- end}}
|
||||||
END IF;
|
END IF;
|
||||||
@@ -49,14 +49,14 @@ BEGIN
|
|||||||
{{- if .AuditDelete}}
|
{{- if .AuditDelete}}
|
||||||
ELSIF TG_OP = 'DELETE' THEN
|
ELSIF TG_OP = 'DELETE' THEN
|
||||||
-- Record DELETE
|
-- Record DELETE
|
||||||
INSERT INTO {{.AuditSchema}}.atevent (tablename, tableprefix, rid_parent, rid_deletedparent, changeuser, changedate, changetime, actionx)
|
INSERT INTO {{quote_ident .AuditSchema}}.atevent (tablename, tableprefix, rid_parent, rid_deletedparent, changeuser, changedate, changetime, actionx)
|
||||||
VALUES ('{{.TableName}}', {{.TablePrefix}}, old.{{.PrimaryKey}}, old.{{.PrimaryKey}}, m_user, CURRENT_DATE, CURRENT_TIME, 3)
|
VALUES ({{quote .TableName}}, {{.TablePrefix}}, old.{{quote_ident .PrimaryKey}}, old.{{quote_ident .PrimaryKey}}, m_user, CURRENT_DATE, CURRENT_TIME, 3)
|
||||||
RETURNING rid_atevent INTO m_atevent;
|
RETURNING rid_atevent INTO m_atevent;
|
||||||
|
|
||||||
-- Record deleted column values
|
-- Record deleted column values
|
||||||
{{- range .DeleteColumns}}
|
{{- range .DeleteColumns}}
|
||||||
INSERT INTO {{$.AuditSchema}}.atdetail(rid_atevent, datacolumn, changedfrom, changedto)
|
INSERT INTO {{quote_ident $.AuditSchema}}.atdetail(rid_atevent, datacolumn, changedfrom, changedto)
|
||||||
VALUES (m_atevent, '{{.Name}}', substr({{.OldValue}}, 1, 1000), NULL);
|
VALUES (m_atevent, {{quote .Name}}, substr({{.OldValue}}, 1, 1000), NULL);
|
||||||
{{- end}}
|
{{- end}}
|
||||||
{{- end}}
|
{{- end}}
|
||||||
END IF;
|
END IF;
|
||||||
@@ -76,9 +76,9 @@ EXCEPTION
|
|||||||
RAISE WARNING 'Audit function % failed: %', m_funcname, SQLERRM;
|
RAISE WARNING 'Audit function % failed: %', m_funcname, SQLERRM;
|
||||||
RETURN NULL;
|
RETURN NULL;
|
||||||
END;
|
END;
|
||||||
$body$
|
{{dollar_delimiter}}
|
||||||
LANGUAGE plpgsql
|
LANGUAGE plpgsql
|
||||||
VOLATILE
|
VOLATILE
|
||||||
SECURITY DEFINER;
|
SECURITY DEFINER;
|
||||||
|
|
||||||
COMMENT ON FUNCTION {{qual_table_raw .SchemaName .FunctionName}}() IS 'Audit trigger function for table {{qual_table_raw .SchemaName .TableName}}';
|
COMMENT ON FUNCTION {{qual_table .SchemaName .FunctionName}}() IS {{quote (printf "Audit trigger function for table %s" (qual_table .SchemaName .TableName))}};
|
||||||
@@ -1,5 +1,5 @@
|
|||||||
-- Audit Event Header Table
|
-- Audit Event Header Table
|
||||||
CREATE TABLE IF NOT EXISTS {{.AuditSchema}}.atevent (
|
CREATE TABLE IF NOT EXISTS {{quote_ident .AuditSchema}}.atevent (
|
||||||
rid_atevent serial PRIMARY KEY,
|
rid_atevent serial PRIMARY KEY,
|
||||||
tablename text NOT NULL,
|
tablename text NOT NULL,
|
||||||
tableprefix text,
|
tableprefix text,
|
||||||
@@ -12,38 +12,38 @@ CREATE TABLE IF NOT EXISTS {{.AuditSchema}}.atevent (
|
|||||||
CONSTRAINT ck_atevent_action CHECK (actionx IN (1, 2, 3))
|
CONSTRAINT ck_atevent_action CHECK (actionx IN (1, 2, 3))
|
||||||
);
|
);
|
||||||
|
|
||||||
CREATE INDEX IF NOT EXISTS idx_atevent_tablename ON {{.AuditSchema}}.atevent(tablename);
|
CREATE INDEX IF NOT EXISTS idx_atevent_tablename ON {{quote_ident .AuditSchema}}.atevent(tablename);
|
||||||
CREATE INDEX IF NOT EXISTS idx_atevent_rid_parent ON {{.AuditSchema}}.atevent(rid_parent);
|
CREATE INDEX IF NOT EXISTS idx_atevent_rid_parent ON {{quote_ident .AuditSchema}}.atevent(rid_parent);
|
||||||
CREATE INDEX IF NOT EXISTS idx_atevent_changedate ON {{.AuditSchema}}.atevent(changedate);
|
CREATE INDEX IF NOT EXISTS idx_atevent_changedate ON {{quote_ident .AuditSchema}}.atevent(changedate);
|
||||||
CREATE INDEX IF NOT EXISTS idx_atevent_changeuser ON {{.AuditSchema}}.atevent(changeuser);
|
CREATE INDEX IF NOT EXISTS idx_atevent_changeuser ON {{quote_ident .AuditSchema}}.atevent(changeuser);
|
||||||
|
|
||||||
COMMENT ON TABLE {{.AuditSchema}}.atevent IS 'Audit trail header table - tracks all data changes';
|
COMMENT ON TABLE {{quote_ident .AuditSchema}}.atevent IS 'Audit trail header table - tracks all data changes';
|
||||||
COMMENT ON COLUMN {{.AuditSchema}}.atevent.rid_atevent IS 'Audit event ID';
|
COMMENT ON COLUMN {{quote_ident .AuditSchema}}.atevent.rid_atevent IS 'Audit event ID';
|
||||||
COMMENT ON COLUMN {{.AuditSchema}}.atevent.tablename IS 'Name of the table that was modified';
|
COMMENT ON COLUMN {{quote_ident .AuditSchema}}.atevent.tablename IS 'Name of the table that was modified';
|
||||||
COMMENT ON COLUMN {{.AuditSchema}}.atevent.rid_parent IS 'Primary key value of the modified record';
|
COMMENT ON COLUMN {{quote_ident .AuditSchema}}.atevent.rid_parent IS 'Primary key value of the modified record';
|
||||||
COMMENT ON COLUMN {{.AuditSchema}}.atevent.rid_deletedparent IS 'Parent reference for deleted records';
|
COMMENT ON COLUMN {{quote_ident .AuditSchema}}.atevent.rid_deletedparent IS 'Parent reference for deleted records';
|
||||||
COMMENT ON COLUMN {{.AuditSchema}}.atevent.changeuser IS 'User who made the change';
|
COMMENT ON COLUMN {{quote_ident .AuditSchema}}.atevent.changeuser IS 'User who made the change';
|
||||||
COMMENT ON COLUMN {{.AuditSchema}}.atevent.changedate IS 'Date of change';
|
COMMENT ON COLUMN {{quote_ident .AuditSchema}}.atevent.changedate IS 'Date of change';
|
||||||
COMMENT ON COLUMN {{.AuditSchema}}.atevent.changetime IS 'Time of change';
|
COMMENT ON COLUMN {{quote_ident .AuditSchema}}.atevent.changetime IS 'Time of change';
|
||||||
COMMENT ON COLUMN {{.AuditSchema}}.atevent.actionx IS 'Action type: 1=INSERT, 2=UPDATE, 3=DELETE';
|
COMMENT ON COLUMN {{quote_ident .AuditSchema}}.atevent.actionx IS 'Action type: 1=INSERT, 2=UPDATE, 3=DELETE';
|
||||||
|
|
||||||
-- Audit Event Detail Table
|
-- Audit Event Detail Table
|
||||||
CREATE TABLE IF NOT EXISTS {{.AuditSchema}}.atdetail (
|
CREATE TABLE IF NOT EXISTS {{quote_ident .AuditSchema}}.atdetail (
|
||||||
rid_atdetail serial PRIMARY KEY,
|
rid_atdetail serial PRIMARY KEY,
|
||||||
rid_atevent integer NOT NULL,
|
rid_atevent integer NOT NULL,
|
||||||
datacolumn text NOT NULL,
|
datacolumn text NOT NULL,
|
||||||
changedfrom text,
|
changedfrom text,
|
||||||
changedto text,
|
changedto text,
|
||||||
CONSTRAINT fk_atdetail_atevent FOREIGN KEY (rid_atevent)
|
CONSTRAINT fk_atdetail_atevent FOREIGN KEY (rid_atevent)
|
||||||
REFERENCES {{.AuditSchema}}.atevent(rid_atevent) ON DELETE CASCADE
|
REFERENCES {{quote_ident .AuditSchema}}.atevent(rid_atevent) ON DELETE CASCADE
|
||||||
);
|
);
|
||||||
|
|
||||||
CREATE INDEX IF NOT EXISTS idx_atdetail_rid_atevent ON {{.AuditSchema}}.atdetail(rid_atevent);
|
CREATE INDEX IF NOT EXISTS idx_atdetail_rid_atevent ON {{quote_ident .AuditSchema}}.atdetail(rid_atevent);
|
||||||
CREATE INDEX IF NOT EXISTS idx_atdetail_datacolumn ON {{.AuditSchema}}.atdetail(datacolumn);
|
CREATE INDEX IF NOT EXISTS idx_atdetail_datacolumn ON {{quote_ident .AuditSchema}}.atdetail(datacolumn);
|
||||||
|
|
||||||
COMMENT ON TABLE {{.AuditSchema}}.atdetail IS 'Audit trail detail table - stores individual column changes';
|
COMMENT ON TABLE {{quote_ident .AuditSchema}}.atdetail IS 'Audit trail detail table - stores individual column changes';
|
||||||
COMMENT ON COLUMN {{.AuditSchema}}.atdetail.rid_atdetail IS 'Audit detail ID';
|
COMMENT ON COLUMN {{quote_ident .AuditSchema}}.atdetail.rid_atdetail IS 'Audit detail ID';
|
||||||
COMMENT ON COLUMN {{.AuditSchema}}.atdetail.rid_atevent IS 'Reference to audit event';
|
COMMENT ON COLUMN {{quote_ident .AuditSchema}}.atdetail.rid_atevent IS 'Reference to audit event';
|
||||||
COMMENT ON COLUMN {{.AuditSchema}}.atdetail.datacolumn IS 'Name of the column that changed';
|
COMMENT ON COLUMN {{quote_ident .AuditSchema}}.atdetail.datacolumn IS 'Name of the column that changed';
|
||||||
COMMENT ON COLUMN {{.AuditSchema}}.atdetail.changedfrom IS 'Old value before change';
|
COMMENT ON COLUMN {{quote_ident .AuditSchema}}.atdetail.changedfrom IS 'Old value before change';
|
||||||
COMMENT ON COLUMN {{.AuditSchema}}.atdetail.changedto IS 'New value after change';
|
COMMENT ON COLUMN {{quote_ident .AuditSchema}}.atdetail.changedto IS 'New value after change';
|
||||||
@@ -1,16 +1,16 @@
|
|||||||
DO $$
|
DO {{dollar_delimiter}}
|
||||||
BEGIN
|
BEGIN
|
||||||
IF NOT EXISTS (
|
IF NOT EXISTS (
|
||||||
SELECT 1
|
SELECT 1
|
||||||
FROM pg_trigger
|
FROM pg_trigger
|
||||||
WHERE tgname = '{{.TriggerName}}'
|
WHERE tgname = {{quote .TriggerName}}
|
||||||
AND tgrelid = '{{qual_table_raw .SchemaName .TableName}}'::regclass
|
AND tgrelid = {{quote (qual_table .SchemaName .TableName)}}::regclass
|
||||||
) THEN
|
) THEN
|
||||||
CREATE TRIGGER {{.TriggerName}}
|
CREATE TRIGGER {{quote_ident .TriggerName}}
|
||||||
AFTER {{.Events}}
|
AFTER {{.Events}}
|
||||||
ON {{qual_table_raw .SchemaName .TableName}}
|
ON {{qual_table .SchemaName .TableName}}
|
||||||
FOR EACH ROW
|
FOR EACH ROW
|
||||||
EXECUTE FUNCTION {{qual_table_raw .SchemaName .FunctionName}}();
|
EXECUTE FUNCTION {{qual_table .SchemaName .FunctionName}}();
|
||||||
END IF;
|
END IF;
|
||||||
END;
|
END;
|
||||||
$$;
|
{{dollar_delimiter}};
|
||||||
@@ -1,39 +1,39 @@
|
|||||||
{{/* Base constraint template */}}
|
{{/* Base constraint template */}}
|
||||||
{{- define "constraint_base" -}}
|
{{- define "constraint_base" -}}
|
||||||
ALTER TABLE {{qual_table_raw .SchemaName .TableName}}
|
ALTER TABLE {{qual_table .SchemaName .TableName}}
|
||||||
ADD CONSTRAINT {{.ConstraintName}}
|
ADD CONSTRAINT {{quote_ident .ConstraintName}}
|
||||||
{{block "constraint_definition" .}}{{end}};
|
{{block "constraint_definition" .}}{{end}};
|
||||||
{{- end -}}
|
{{- end -}}
|
||||||
|
|
||||||
{{/* Drop constraint with check */}}
|
{{/* Drop constraint with check */}}
|
||||||
{{- define "drop_constraint_safe" -}}
|
{{- define "drop_constraint_safe" -}}
|
||||||
DO $$
|
DO {{dollar_delimiter}}
|
||||||
BEGIN
|
BEGIN
|
||||||
IF EXISTS (
|
IF EXISTS (
|
||||||
SELECT 1 FROM information_schema.table_constraints
|
SELECT 1 FROM information_schema.table_constraints
|
||||||
WHERE table_schema = '{{.SchemaName}}'
|
WHERE table_schema = {{quote .SchemaName}}
|
||||||
AND table_name = '{{.TableName}}'
|
AND table_name = {{quote .TableName}}
|
||||||
AND constraint_name = '{{.ConstraintName}}'
|
AND constraint_name = {{quote .ConstraintName}}
|
||||||
) THEN
|
) THEN
|
||||||
ALTER TABLE {{qual_table_raw .SchemaName .TableName}}
|
ALTER TABLE {{qual_table .SchemaName .TableName}}
|
||||||
DROP CONSTRAINT {{.ConstraintName}};
|
DROP CONSTRAINT {{quote_ident .ConstraintName}};
|
||||||
END IF;
|
END IF;
|
||||||
END;
|
END;
|
||||||
$$;
|
{{dollar_delimiter}};
|
||||||
{{- end -}}
|
{{- end -}}
|
||||||
|
|
||||||
{{/* Add constraint with existence check */}}
|
{{/* Add constraint with existence check */}}
|
||||||
{{- define "add_constraint_safe" -}}
|
{{- define "add_constraint_safe" -}}
|
||||||
DO $$
|
DO {{dollar_delimiter}}
|
||||||
BEGIN
|
BEGIN
|
||||||
IF NOT EXISTS (
|
IF NOT EXISTS (
|
||||||
SELECT 1 FROM information_schema.table_constraints
|
SELECT 1 FROM information_schema.table_constraints
|
||||||
WHERE table_schema = '{{.SchemaName}}'
|
WHERE table_schema = {{quote .SchemaName}}
|
||||||
AND table_name = '{{.TableName}}'
|
AND table_name = {{quote .TableName}}
|
||||||
AND constraint_name = '{{.ConstraintName}}'
|
AND constraint_name = {{quote .ConstraintName}}
|
||||||
) THEN
|
) THEN
|
||||||
{{template "constraint_base" .}}
|
{{template "constraint_base" .}}
|
||||||
END IF;
|
END IF;
|
||||||
END;
|
END;
|
||||||
$$;
|
{{dollar_delimiter}};
|
||||||
{{- end -}}
|
{{- end -}}
|
||||||
@@ -1,23 +1,23 @@
|
|||||||
{{/* Base DDL template with common structure */}}
|
{{/* Base DDL template with common structure */}}
|
||||||
{{- define "ddl_header" -}}
|
{{- define "ddl_header" -}}
|
||||||
-- DDL Operation: {{.Operation}}
|
-- DDL Operation: {{comment_text .Operation}}
|
||||||
-- Schema: {{.Schema}}
|
-- Schema: {{comment_text .Schema}}
|
||||||
-- Object: {{.ObjectName}}
|
-- Object: {{comment_text .ObjectName}}
|
||||||
{{- end -}}
|
{{- end -}}
|
||||||
|
|
||||||
{{- define "ddl_footer" -}}
|
{{- define "ddl_footer" -}}
|
||||||
-- End of {{.Operation}}
|
-- End of {{comment_text .Operation}}
|
||||||
{{- end -}}
|
{{- end -}}
|
||||||
|
|
||||||
{{/* Base ALTER TABLE structure */}}
|
{{/* Base ALTER TABLE structure */}}
|
||||||
{{- define "alter_table_base" -}}
|
{{- define "alter_table_base" -}}
|
||||||
ALTER TABLE {{qual_table_raw .SchemaName .TableName}}
|
ALTER TABLE {{qual_table .SchemaName .TableName}}
|
||||||
{{block "alter_operation" .}}{{end}};
|
{{block "alter_operation" .}}{{end}};
|
||||||
{{- end -}}
|
{{- end -}}
|
||||||
|
|
||||||
{{/* Common existence check pattern */}}
|
{{/* Common existence check pattern */}}
|
||||||
{{- define "exists_check" -}}
|
{{- define "exists_check" -}}
|
||||||
DO $$
|
DO {{dollar_delimiter}}
|
||||||
BEGIN
|
BEGIN
|
||||||
IF NOT EXISTS (
|
IF NOT EXISTS (
|
||||||
{{block "exists_query" .}}{{end}}
|
{{block "exists_query" .}}{{end}}
|
||||||
@@ -25,10 +25,10 @@ BEGIN
|
|||||||
{{block "create_statement" .}}{{end}}
|
{{block "create_statement" .}}{{end}}
|
||||||
END IF;
|
END IF;
|
||||||
END;
|
END;
|
||||||
$$;
|
{{dollar_delimiter}};
|
||||||
{{- end -}}
|
{{- end -}}
|
||||||
|
|
||||||
{{/* Common drop pattern */}}
|
{{/* Common drop pattern */}}
|
||||||
{{- define "drop_if_exists" -}}
|
{{- define "drop_if_exists" -}}
|
||||||
{{block "drop_type" .}}{{end}} IF EXISTS {{qual_table_raw .SchemaName .ObjectName}};
|
{{block "drop_type" .}}{{end}} IF EXISTS {{qual_table .SchemaName .ObjectName}};
|
||||||
{{- end -}}
|
{{- end -}}
|
||||||
@@ -1 +1 @@
|
|||||||
COMMENT ON COLUMN {{qual_table .SchemaName .TableName}}.{{quote_ident .ColumnName}} IS '{{.Comment}}';
|
COMMENT ON COLUMN {{qual_table .SchemaName .TableName}}.{{quote_ident .ColumnName}} IS {{quote .Comment}};
|
||||||
@@ -1 +1 @@
|
|||||||
COMMENT ON TABLE {{qual_table .SchemaName .TableName}} IS '{{.Comment}}';
|
COMMENT ON TABLE {{qual_table .SchemaName .TableName}} IS {{quote .Comment}};
|
||||||
@@ -1,12 +1,12 @@
|
|||||||
DO $$
|
DO {{dollar_delimiter}}
|
||||||
BEGIN
|
BEGIN
|
||||||
IF NOT EXISTS (
|
IF NOT EXISTS (
|
||||||
SELECT 1 FROM information_schema.table_constraints
|
SELECT 1 FROM information_schema.table_constraints
|
||||||
WHERE table_schema = '{{.SchemaName}}'
|
WHERE table_schema = {{quote .SchemaName}}
|
||||||
AND table_name = '{{.TableName}}'
|
AND table_name = {{quote .TableName}}
|
||||||
AND constraint_name = '{{.ConstraintName}}'
|
AND constraint_name = {{quote .ConstraintName}}
|
||||||
) THEN
|
) THEN
|
||||||
ALTER TABLE {{qual_table .SchemaName .TableName}} ADD CONSTRAINT {{quote_ident .ConstraintName}} CHECK ({{.Expression}});
|
ALTER TABLE {{qual_table .SchemaName .TableName}} ADD CONSTRAINT {{quote_ident .ConstraintName}} CHECK ({{.Expression}});
|
||||||
END IF;
|
END IF;
|
||||||
END;
|
END;
|
||||||
$$;
|
{{dollar_delimiter}};
|
||||||
@@ -1,10 +1,10 @@
|
|||||||
DO $$
|
DO {{dollar_delimiter}}
|
||||||
BEGIN
|
BEGIN
|
||||||
IF NOT EXISTS (
|
IF NOT EXISTS (
|
||||||
SELECT 1 FROM information_schema.table_constraints
|
SELECT 1 FROM information_schema.table_constraints
|
||||||
WHERE table_schema = '{{.SchemaName}}'
|
WHERE table_schema = {{quote .SchemaName}}
|
||||||
AND table_name = '{{.TableName}}'
|
AND table_name = {{quote .TableName}}
|
||||||
AND constraint_name = '{{.ConstraintName}}'
|
AND constraint_name = {{quote .ConstraintName}}
|
||||||
) THEN
|
) THEN
|
||||||
ALTER TABLE {{qual_table .SchemaName .TableName}}
|
ALTER TABLE {{qual_table .SchemaName .TableName}}
|
||||||
ADD CONSTRAINT {{quote_ident .ConstraintName}}
|
ADD CONSTRAINT {{quote_ident .ConstraintName}}
|
||||||
@@ -15,4 +15,4 @@ BEGIN
|
|||||||
{{.Deferral}}{{end}};
|
{{.Deferral}}{{end}};
|
||||||
END IF;
|
END IF;
|
||||||
END;
|
END;
|
||||||
$$;
|
{{dollar_delimiter}};
|
||||||
@@ -1,13 +1,13 @@
|
|||||||
DO $$
|
DO {{dollar_delimiter}}
|
||||||
BEGIN
|
BEGIN
|
||||||
IF NOT EXISTS (
|
IF NOT EXISTS (
|
||||||
SELECT 1 FROM information_schema.table_constraints
|
SELECT 1 FROM information_schema.table_constraints
|
||||||
WHERE table_schema = '{{.SchemaName}}'
|
WHERE table_schema = {{quote .SchemaName}}
|
||||||
AND table_name = '{{.TableName}}'
|
AND table_name = {{quote .TableName}}
|
||||||
AND constraint_name = '{{.ConstraintName}}'
|
AND constraint_name = {{quote .ConstraintName}}
|
||||||
) THEN
|
) THEN
|
||||||
ALTER TABLE {{qual_table .SchemaName .TableName}}
|
ALTER TABLE {{qual_table .SchemaName .TableName}}
|
||||||
ADD CONSTRAINT {{quote_ident .ConstraintName}} PRIMARY KEY ({{.Columns}});
|
ADD CONSTRAINT {{quote_ident .ConstraintName}} PRIMARY KEY ({{.Columns}});
|
||||||
END IF;
|
END IF;
|
||||||
END;
|
END;
|
||||||
$$;
|
{{dollar_delimiter}};
|
||||||
@@ -1,4 +1,4 @@
|
|||||||
DO $$
|
DO {{dollar_delimiter}}
|
||||||
DECLARE
|
DECLARE
|
||||||
current_pk_name text;
|
current_pk_name text;
|
||||||
current_pk_matches boolean := false;
|
current_pk_matches boolean := false;
|
||||||
@@ -16,22 +16,22 @@ BEGIN
|
|||||||
ON a.attrelid = t.oid
|
ON a.attrelid = t.oid
|
||||||
AND a.attnum = cols.attnum
|
AND a.attnum = cols.attnum
|
||||||
WHERE c.contype = 'p'
|
WHERE c.contype = 'p'
|
||||||
AND n.nspname = '{{.SchemaName}}'
|
AND n.nspname = {{quote .SchemaName}}
|
||||||
AND t.relname = '{{.TableName}}'
|
AND t.relname = {{quote .TableName}}
|
||||||
ORDER BY cols.ord
|
ORDER BY cols.ord
|
||||||
),
|
),
|
||||||
ARRAY[]::text[]
|
ARRAY[]::text[]
|
||||||
) = ARRAY[{{.ColumnNames}}]
|
) = ARRAY[{{.ColumnNames}}]
|
||||||
INTO current_pk_name, current_pk_matches
|
INTO current_pk_name, current_pk_matches
|
||||||
FROM information_schema.table_constraints tc
|
FROM information_schema.table_constraints tc
|
||||||
WHERE tc.table_schema = '{{.SchemaName}}'
|
WHERE tc.table_schema = {{quote .SchemaName}}
|
||||||
AND tc.table_name = '{{.TableName}}'
|
AND tc.table_name = {{quote .TableName}}
|
||||||
AND tc.constraint_type = 'PRIMARY KEY';
|
AND tc.constraint_type = 'PRIMARY KEY';
|
||||||
|
|
||||||
IF current_pk_name IS NOT NULL
|
IF current_pk_name IS NOT NULL
|
||||||
AND NOT current_pk_matches
|
AND NOT current_pk_matches
|
||||||
AND current_pk_name IN ({{.AutoGenNames}}) THEN
|
AND current_pk_name IN ({{.AutoGenNames}}) THEN
|
||||||
EXECUTE 'ALTER TABLE {{qual_table .SchemaName .TableName}} DROP CONSTRAINT ' || quote_ident(current_pk_name) || ' CASCADE';
|
EXECUTE {{quote (printf "ALTER TABLE %s DROP CONSTRAINT " (qual_table .SchemaName .TableName))}} || quote_ident(current_pk_name) || ' CASCADE';
|
||||||
END IF;
|
END IF;
|
||||||
|
|
||||||
-- Add the desired primary key only when no matching primary key already exists.
|
-- Add the desired primary key only when no matching primary key already exists.
|
||||||
@@ -40,4 +40,4 @@ BEGIN
|
|||||||
ALTER TABLE {{qual_table .SchemaName .TableName}} ADD CONSTRAINT {{quote_ident .ConstraintName}} PRIMARY KEY ({{.Columns}});
|
ALTER TABLE {{qual_table .SchemaName .TableName}} ADD CONSTRAINT {{quote_ident .ConstraintName}} PRIMARY KEY ({{.Columns}});
|
||||||
END IF;
|
END IF;
|
||||||
END;
|
END;
|
||||||
$$;
|
{{dollar_delimiter}};
|
||||||
|
|||||||
@@ -1,12 +1,12 @@
|
|||||||
DO $$
|
DO {{dollar_delimiter}}
|
||||||
BEGIN
|
BEGIN
|
||||||
IF NOT EXISTS (
|
IF NOT EXISTS (
|
||||||
SELECT 1 FROM information_schema.table_constraints
|
SELECT 1 FROM information_schema.table_constraints
|
||||||
WHERE table_schema = '{{.SchemaName}}'
|
WHERE table_schema = {{quote .SchemaName}}
|
||||||
AND table_name = '{{.TableName}}'
|
AND table_name = {{quote .TableName}}
|
||||||
AND constraint_name = '{{.ConstraintName}}'
|
AND constraint_name = {{quote .ConstraintName}}
|
||||||
) THEN
|
) THEN
|
||||||
ALTER TABLE {{qual_table .SchemaName .TableName}} ADD CONSTRAINT {{quote_ident .ConstraintName}} UNIQUE {{if .UsingIndex}}USING INDEX {{quote_ident .UsingIndex}}{{else}}({{.Columns}}){{end}};
|
ALTER TABLE {{qual_table .SchemaName .TableName}} ADD CONSTRAINT {{quote_ident .ConstraintName}} UNIQUE {{if .UsingIndex}}USING INDEX {{quote_ident .UsingIndex}}{{else}}({{.Columns}}){{end}};
|
||||||
END IF;
|
END IF;
|
||||||
END;
|
END;
|
||||||
$$;
|
{{dollar_delimiter}};
|
||||||
|
|||||||
@@ -2,7 +2,7 @@
|
|||||||
|
|
||||||
{{/* Column definition fragment */}}
|
{{/* Column definition fragment */}}
|
||||||
{{- define "column_definition" -}}
|
{{- define "column_definition" -}}
|
||||||
{{.Name}} {{.Type}}
|
{{quote_ident .Name}} {{.Type}}
|
||||||
{{- if .Generated}} GENERATED ALWAYS AS ({{.Generated}}) STORED{{end}}
|
{{- if .Generated}} GENERATED ALWAYS AS ({{.Generated}}) STORED{{end}}
|
||||||
{{- if .Identity}} GENERATED {{.Identity}} AS IDENTITY{{end}}
|
{{- if .Identity}} GENERATED {{.Identity}} AS IDENTITY{{end}}
|
||||||
{{- if and .Default (not .Generated) (not .Identity)}} DEFAULT {{.Default}}{{end}}
|
{{- if and .Default (not .Generated) (not .Identity)}} DEFAULT {{.Default}}{{end}}
|
||||||
@@ -18,7 +18,7 @@
|
|||||||
|
|
||||||
{{/* Qualified table name */}}
|
{{/* Qualified table name */}}
|
||||||
{{- define "qualified_table" -}}
|
{{- define "qualified_table" -}}
|
||||||
{{qual_table_raw .SchemaName .TableName}}
|
{{qual_table .SchemaName .TableName}}
|
||||||
{{- end -}}
|
{{- end -}}
|
||||||
|
|
||||||
{{/* Index method clause */}}
|
{{/* Index method clause */}}
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
DO $$
|
DO {{dollar_delimiter}}
|
||||||
DECLARE
|
DECLARE
|
||||||
m_cnt bigint;
|
m_cnt bigint;
|
||||||
m_next bigint;
|
m_next bigint;
|
||||||
@@ -6,8 +6,8 @@ BEGIN
|
|||||||
IF EXISTS (
|
IF EXISTS (
|
||||||
SELECT 1 FROM pg_class c
|
SELECT 1 FROM pg_class c
|
||||||
INNER JOIN pg_namespace n ON n.oid = c.relnamespace
|
INNER JOIN pg_namespace n ON n.oid = c.relnamespace
|
||||||
WHERE c.relname = '{{.SequenceName}}'
|
WHERE c.relname = {{quote .SequenceName}}
|
||||||
AND n.nspname = '{{.SchemaName}}'
|
AND n.nspname = {{quote .SchemaName}}
|
||||||
AND c.relkind = 'S'
|
AND c.relkind = 'S'
|
||||||
) THEN
|
) THEN
|
||||||
SELECT COALESCE(MAX({{quote_ident .ColumnName}}), 0) + 1
|
SELECT COALESCE(MAX({{quote_ident .ColumnName}}), 0) + 1
|
||||||
@@ -20,8 +20,8 @@ BEGIN
|
|||||||
|
|
||||||
-- Only move the sequence forward; never rewind one that is already past the data.
|
-- Only move the sequence forward; never rewind one that is already past the data.
|
||||||
IF m_cnt > m_next THEN
|
IF m_cnt > m_next THEN
|
||||||
PERFORM setval('{{qual_table_raw .SchemaName .SequenceName}}'::regclass, m_cnt, false);
|
PERFORM setval({{quote (qual_table .SchemaName .SequenceName)}}::regclass, m_cnt, false);
|
||||||
END IF;
|
END IF;
|
||||||
END IF;
|
END IF;
|
||||||
END;
|
END;
|
||||||
$$;
|
{{dollar_delimiter}};
|
||||||
@@ -0,0 +1,91 @@
|
|||||||
|
package pgsql
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"github.com/jackc/pgx/v5"
|
||||||
|
|
||||||
|
"git.warky.dev/wdevs/relspecgo/pkg/models"
|
||||||
|
)
|
||||||
|
|
||||||
|
// conversionData accepts executable SQL only through explicitly named column metadata.
|
||||||
|
func conversionData(schema, table string, col *models.Column) (AlterColumnTypeWithCheckData, error) {
|
||||||
|
target := effectiveAlterColumnSQLType(col)
|
||||||
|
data := AlterColumnTypeWithCheckData{
|
||||||
|
SchemaName: schema, TableName: table, ColumnName: col.Name,
|
||||||
|
NewType: target, EquivalentTypes: equivalentTypeListSQL(target),
|
||||||
|
UsingExpr: buildAlterColumnUsingExpression(col.Name, target),
|
||||||
|
}
|
||||||
|
for key, destination := range map[string]*string{"pgsql_using": &data.UsingExpr, "pgsql_preflight": &data.PreflightPredicate} {
|
||||||
|
if value, exists := col.Metadata[key]; exists {
|
||||||
|
expression, ok := value.(string)
|
||||||
|
if !ok || strings.TrimSpace(expression) == "" {
|
||||||
|
return data, fmt.Errorf("%s.%s.%s metadata %s must be a nonempty SQL expression", schema, table, col.Name, key)
|
||||||
|
}
|
||||||
|
*destination = expression
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return data, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// sqlLiteral is independent of the connection's standard_conforming_strings setting.
|
||||||
|
func sqlLiteral(value string) string {
|
||||||
|
return "E'" + strings.NewReplacer("\\", "\\\\", "'", "''").Replace(value) + "'"
|
||||||
|
}
|
||||||
|
|
||||||
|
func dollarBlock(body, prefix string) string {
|
||||||
|
for i := 0; ; i++ {
|
||||||
|
delimiter := fmt.Sprintf("$%s_%d$", prefix, i)
|
||||||
|
if !strings.Contains(body, delimiter) {
|
||||||
|
return "DO " + delimiter + "\n" + body + "\n" + delimiter + ";"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// preflightColumnType checks every value before any schema changes. It is advisory:
|
||||||
|
// writes can race the check, so the ALTER itself still performs the real conversion.
|
||||||
|
func preflightColumnType(data AlterColumnTypeWithCheckData, flatten bool) string {
|
||||||
|
ident := pgx.Identifier{data.TableName}
|
||||||
|
if data.SchemaName != "" {
|
||||||
|
if flatten {
|
||||||
|
ident = pgx.Identifier{data.SchemaName + "_" + data.TableName}
|
||||||
|
} else {
|
||||||
|
ident = pgx.Identifier{data.SchemaName, data.TableName}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
table := ident.Sanitize()
|
||||||
|
body := fmt.Sprintf(`DECLARE current_type text;
|
||||||
|
BEGIN
|
||||||
|
SELECT pg_catalog.format_type(atttypid, atttypmod) INTO current_type
|
||||||
|
FROM pg_catalog.pg_attribute
|
||||||
|
WHERE attrelid = to_regclass(%s) AND attname = %s AND attnum > 0 AND NOT attisdropped;
|
||||||
|
IF current_type IS NOT NULL AND current_type <> ALL(ARRAY[%s]) THEN
|
||||||
|
`, sqlLiteral(table), sqlLiteral(data.ColumnName), data.EquivalentTypes)
|
||||||
|
if data.PreflightPredicate != "" {
|
||||||
|
body += fmt.Sprintf(` IF EXISTS (SELECT 1 FROM %s WHERE NOT COALESCE((%s), false)) THEN
|
||||||
|
RAISE EXCEPTION USING MESSAGE = %s;
|
||||||
|
END IF;
|
||||||
|
`, table, data.PreflightPredicate, sqlLiteral("type conversion preflight rejected rows for "+data.SchemaName+"."+data.TableName+"."+data.ColumnName))
|
||||||
|
}
|
||||||
|
// Aggregate evaluation forces the conversion for all rows, including non-first rows.
|
||||||
|
body += fmt.Sprintf(" PERFORM count((%s)::%s) FROM %s;\n END IF;\nEND;", data.UsingExpr, data.NewType, table)
|
||||||
|
return dollarBlock(body, "relspec_preflight")
|
||||||
|
}
|
||||||
|
|
||||||
|
func (w *Writer) generateTypePreflights(schema *models.Schema) ([]string, error) {
|
||||||
|
var statements []string
|
||||||
|
for _, table := range schema.Tables {
|
||||||
|
for _, col := range getSortedColumns(table.Columns) {
|
||||||
|
if col.Generated || col.Identity {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
data, err := conversionData(schema.Name, table.Name, col)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
statements = append(statements, preflightColumnType(data, w.options.FlattenSchema))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return statements, nil
|
||||||
|
}
|
||||||
@@ -0,0 +1,72 @@
|
|||||||
|
package pgsql
|
||||||
|
|
||||||
|
import (
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"git.warky.dev/wdevs/relspecgo/pkg/models"
|
||||||
|
"git.warky.dev/wdevs/relspecgo/pkg/writers"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestConversionMetadataValidationAndBlockDelimiters(t *testing.T) {
|
||||||
|
col := models.InitColumn("name", "accounts", "public")
|
||||||
|
col.Type = "integer"
|
||||||
|
for _, key := range []string{"pgsql_using", "pgsql_preflight"} {
|
||||||
|
for _, value := range []interface{}{true, " "} {
|
||||||
|
col.Metadata[key] = value
|
||||||
|
if _, err := conversionData("public", "accounts", col); err == nil || !strings.Contains(err.Error(), key) {
|
||||||
|
t.Fatalf("expected invalid %s metadata error: %v", key, err)
|
||||||
|
}
|
||||||
|
delete(col.Metadata, key)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
col.Metadata["pgsql_using"] = "length('$relspec_0$ $relspec_preflight_0$')"
|
||||||
|
data, err := conversionData("public", "accounts", col)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
executor, err := NewTemplateExecutor(false)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
alter, err := executor.ExecuteAlterColumnTypeWithCheck(data)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if !strings.HasPrefix(alter, "DO $relspec_1$") {
|
||||||
|
t.Fatalf("unsafe alter delimiter: %s", alter)
|
||||||
|
}
|
||||||
|
preflight := preflightColumnType(data, false)
|
||||||
|
if !strings.HasPrefix(preflight, "DO $relspec_preflight_1$") {
|
||||||
|
t.Fatalf("unsafe preflight delimiter: %s", preflight)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPreflightsPrecedeAllSchemaMutations(t *testing.T) {
|
||||||
|
db := models.InitDatabase("preflight")
|
||||||
|
for _, name := range []string{"one", "two"} {
|
||||||
|
schema := models.InitSchema(name)
|
||||||
|
table := models.InitTable("accounts", name)
|
||||||
|
col := models.InitColumn("name", "accounts", name)
|
||||||
|
col.Type = "integer"
|
||||||
|
table.Columns[col.Name] = col
|
||||||
|
schema.Tables = append(schema.Tables, table)
|
||||||
|
db.Schemas = append(db.Schemas, schema)
|
||||||
|
}
|
||||||
|
stmts, err := NewWriter(&writers.WriterOptions{}).GenerateDatabaseStatements(db)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
lastPreflight, firstDDL := -1, len(stmts)
|
||||||
|
for i, stmt := range stmts {
|
||||||
|
if strings.HasPrefix(stmt, "DO $relspec_preflight") {
|
||||||
|
lastPreflight = i
|
||||||
|
}
|
||||||
|
if strings.HasPrefix(stmt, "CREATE SCHEMA") && i < firstDDL {
|
||||||
|
firstDDL = i
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if lastPreflight < 0 || lastPreflight >= firstDDL {
|
||||||
|
t.Fatal("preflights must precede all schema changes")
|
||||||
|
}
|
||||||
|
}
|
||||||
+310
-291
File diff suppressed because it is too large
Load Diff
@@ -22,7 +22,7 @@ func TestWriteDatabase(t *testing.T) {
|
|||||||
idCol := models.InitColumn("id", "users", "public")
|
idCol := models.InitColumn("id", "users", "public")
|
||||||
idCol.Type = "integer"
|
idCol.Type = "integer"
|
||||||
idCol.Description = "Primary key"
|
idCol.Description = "Primary key"
|
||||||
idCol.Default = "nextval('public.identity_users_id'::regclass)"
|
idCol.Default = "nextval('\"public\".\"identity_users_id\"'::regclass)"
|
||||||
table.Columns["id"] = idCol
|
table.Columns["id"] = idCol
|
||||||
|
|
||||||
nameCol := models.InitColumn("name", "users", "public")
|
nameCol := models.InitColumn("name", "users", "public")
|
||||||
@@ -117,7 +117,7 @@ func TestWriteDatabase_ConcurrentIndex(t *testing.T) {
|
|||||||
|
|
||||||
output := buf.String()
|
output := buf.String()
|
||||||
|
|
||||||
if !strings.Contains(output, "CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_users_email") {
|
if !strings.Contains(output, "CREATE INDEX CONCURRENTLY IF NOT EXISTS \"idx_users_email\"") {
|
||||||
t.Errorf("Output missing CONCURRENTLY index creation:\n%s", output)
|
t.Errorf("Output missing CONCURRENTLY index creation:\n%s", output)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -151,7 +151,7 @@ func TestWriteDatabase_GinIndexOnTextArrayDoesNotUseTrigramOperatorClass(t *test
|
|||||||
}
|
}
|
||||||
|
|
||||||
output := buf.String()
|
output := buf.String()
|
||||||
if !strings.Contains(output, `USING gin (tags array_ops)`) {
|
if !strings.Contains(output, `USING gin ("tags" array_ops)`) {
|
||||||
t.Fatalf("expected GIN index on array column with array_ops, got:\n%s", output)
|
t.Fatalf("expected GIN index on array column with array_ops, got:\n%s", output)
|
||||||
}
|
}
|
||||||
if strings.Contains(output, "gin_trgm_ops") {
|
if strings.Contains(output, "gin_trgm_ops") {
|
||||||
@@ -172,7 +172,7 @@ func TestWriteDatabase_GinIndexOnQuotedTextColumnUsesTrigramOperatorClass(t *tes
|
|||||||
index := &models.Index{
|
index := &models.Index{
|
||||||
Name: "idx_agent_personas_name_gin",
|
Name: "idx_agent_personas_name_gin",
|
||||||
Type: "gin",
|
Type: "gin",
|
||||||
Columns: []string{`"name"`},
|
Columns: []string{"name"},
|
||||||
}
|
}
|
||||||
table.Indexes[index.Name] = index
|
table.Indexes[index.Name] = index
|
||||||
|
|
||||||
@@ -191,7 +191,7 @@ func TestWriteDatabase_GinIndexOnQuotedTextColumnUsesTrigramOperatorClass(t *tes
|
|||||||
if !strings.Contains(output, `CREATE EXTENSION IF NOT EXISTS pg_trgm`) {
|
if !strings.Contains(output, `CREATE EXTENSION IF NOT EXISTS pg_trgm`) {
|
||||||
t.Fatalf("expected trigram extension for text GIN index, got:\n%s", output)
|
t.Fatalf("expected trigram extension for text GIN index, got:\n%s", output)
|
||||||
}
|
}
|
||||||
if !strings.Contains(output, `USING gin (name gin_trgm_ops)`) {
|
if !strings.Contains(output, `USING gin ("name" gin_trgm_ops)`) {
|
||||||
t.Fatalf("expected quoted text GIN index to include gin_trgm_ops, got:\n%s", output)
|
t.Fatalf("expected quoted text GIN index to include gin_trgm_ops, got:\n%s", output)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -225,7 +225,7 @@ func TestWriteDatabase_GinIndexOnJSONBUsesJSONBOperatorClass(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
output := buf.String()
|
output := buf.String()
|
||||||
if !strings.Contains(output, `USING gin (details jsonb_ops)`) {
|
if !strings.Contains(output, `USING gin ("details" jsonb_ops)`) {
|
||||||
t.Fatalf("expected GIN jsonb index to include jsonb_ops, got:\n%s", output)
|
t.Fatalf("expected GIN jsonb index to include jsonb_ops, got:\n%s", output)
|
||||||
}
|
}
|
||||||
if strings.Contains(output, "gin_trgm_ops") {
|
if strings.Contains(output, "gin_trgm_ops") {
|
||||||
@@ -372,10 +372,10 @@ func TestWriteUniqueConstraints(t *testing.T) {
|
|||||||
if !strings.Contains(output, "-- Unique constraints for schema: public") {
|
if !strings.Contains(output, "-- Unique constraints for schema: public") {
|
||||||
t.Errorf("Output missing unique constraints header")
|
t.Errorf("Output missing unique constraints header")
|
||||||
}
|
}
|
||||||
if !strings.Contains(output, "ADD CONSTRAINT uq_email UNIQUE (email)") {
|
if !strings.Contains(output, "ADD CONSTRAINT \"uq_email\" UNIQUE (\"email\")") {
|
||||||
t.Errorf("Output missing uq_email unique constraint\nFull output:\n%s", output)
|
t.Errorf("Output missing uq_email unique constraint\nFull output:\n%s", output)
|
||||||
}
|
}
|
||||||
if !strings.Contains(output, "ADD CONSTRAINT uq_guid UNIQUE (guid)") {
|
if !strings.Contains(output, "ADD CONSTRAINT \"uq_guid\" UNIQUE (\"guid\")") {
|
||||||
t.Errorf("Output missing uq_guid unique constraint\nFull output:\n%s", output)
|
t.Errorf("Output missing uq_guid unique constraint\nFull output:\n%s", output)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -453,13 +453,13 @@ func TestWriteCheckConstraints(t *testing.T) {
|
|||||||
if !strings.Contains(output, "-- Check constraints for schema: public") {
|
if !strings.Contains(output, "-- Check constraints for schema: public") {
|
||||||
t.Errorf("Output missing check constraints header")
|
t.Errorf("Output missing check constraints header")
|
||||||
}
|
}
|
||||||
if !strings.Contains(output, "ADD CONSTRAINT ck_price_positive CHECK (price >= 0)") {
|
if !strings.Contains(output, "ADD CONSTRAINT \"ck_price_positive\" CHECK (price >= 0)") {
|
||||||
t.Errorf("Output missing ck_price_positive check constraint\nFull output:\n%s", output)
|
t.Errorf("Output missing ck_price_positive check constraint\nFull output:\n%s", output)
|
||||||
}
|
}
|
||||||
if !strings.Contains(output, "ADD CONSTRAINT ck_status_valid CHECK (status IN ('active', 'inactive', 'discontinued'))") {
|
if !strings.Contains(output, "ADD CONSTRAINT \"ck_status_valid\" CHECK (status IN ('active', 'inactive', 'discontinued'))") {
|
||||||
t.Errorf("Output missing ck_status_valid check constraint\nFull output:\n%s", output)
|
t.Errorf("Output missing ck_status_valid check constraint\nFull output:\n%s", output)
|
||||||
}
|
}
|
||||||
if !strings.Contains(output, "ADD CONSTRAINT ck_quantity_nonnegative CHECK (quantity >= 0)") {
|
if !strings.Contains(output, "ADD CONSTRAINT \"ck_quantity_nonnegative\" CHECK (quantity >= 0)") {
|
||||||
t.Errorf("Output missing ck_quantity_nonnegative check constraint\nFull output:\n%s", output)
|
t.Errorf("Output missing ck_quantity_nonnegative check constraint\nFull output:\n%s", output)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -573,9 +573,9 @@ func TestWriteAllConstraintTypes(t *testing.T) {
|
|||||||
// Verify all constraint types are present
|
// Verify all constraint types are present
|
||||||
expectedConstraints := map[string]string{
|
expectedConstraints := map[string]string{
|
||||||
"Primary Key": "PRIMARY KEY",
|
"Primary Key": "PRIMARY KEY",
|
||||||
"Unique": "ADD CONSTRAINT uq_order_number UNIQUE (order_number)",
|
"Unique": "ADD CONSTRAINT \"uq_order_number\" UNIQUE (\"order_number\")",
|
||||||
"Check (total)": "ADD CONSTRAINT ck_total_positive CHECK (total > 0)",
|
"Check (total)": "ADD CONSTRAINT \"ck_total_positive\" CHECK (total > 0)",
|
||||||
"Check (status)": "ADD CONSTRAINT ck_status_valid CHECK (status IN ('pending', 'completed', 'cancelled'))",
|
"Check (status)": "ADD CONSTRAINT \"ck_status_valid\" CHECK (status IN ('pending', 'completed', 'cancelled'))",
|
||||||
"Foreign Key": "FOREIGN KEY",
|
"Foreign Key": "FOREIGN KEY",
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -640,7 +640,7 @@ func TestEscapeQuote(t *testing.T) {
|
|||||||
input string
|
input string
|
||||||
expected string
|
expected string
|
||||||
}{
|
}{
|
||||||
{"simple text", "simple text"},
|
{"\"simple\" text", "\"simple\" text"},
|
||||||
{"text with 'quote'", "text with ''quote''"},
|
{"text with 'quote'", "text with ''quote''"},
|
||||||
{"multiple 'quotes' here", "multiple ''quotes'' here"},
|
{"multiple 'quotes' here", "multiple ''quotes'' here"},
|
||||||
{"", ""},
|
{"", ""},
|
||||||
@@ -1016,15 +1016,9 @@ func TestGenerateAddColumnStatements(t *testing.T) {
|
|||||||
|
|
||||||
// Verify expected elements
|
// Verify expected elements
|
||||||
expectedStrings := []string{
|
expectedStrings := []string{
|
||||||
"ALTER TABLE public.users ADD COLUMN id integer NOT NULL",
|
"ALTER TABLE \"public\".\"users\" ADD COLUMN IF NOT EXISTS \"id\" integer NOT NULL",
|
||||||
"ALTER TABLE public.users ADD COLUMN email varchar(255) NOT NULL",
|
"ALTER TABLE \"public\".\"users\" ADD COLUMN IF NOT EXISTS \"email\" varchar(255) NOT NULL",
|
||||||
"ALTER TABLE public.users ADD COLUMN status text DEFAULT 'active'",
|
"ALTER TABLE \"public\".\"users\" ADD COLUMN IF NOT EXISTS \"status\" text DEFAULT 'active'",
|
||||||
"information_schema.columns",
|
|
||||||
"table_schema = 'public'",
|
|
||||||
"table_name = 'users'",
|
|
||||||
"column_name = 'id'",
|
|
||||||
"column_name = 'email'",
|
|
||||||
"column_name = 'status'",
|
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, expected := range expectedStrings {
|
for _, expected := range expectedStrings {
|
||||||
@@ -1033,12 +1027,6 @@ func TestGenerateAddColumnStatements(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Verify DO blocks are present for conditional adds
|
|
||||||
doBlockCount := strings.Count(output, "DO $$")
|
|
||||||
if doBlockCount < 3 {
|
|
||||||
t.Errorf("Expected at least 3 DO blocks (one per column), got %d", doBlockCount)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Verify IF NOT EXISTS logic
|
// Verify IF NOT EXISTS logic
|
||||||
ifNotExistsCount := strings.Count(output, "IF NOT EXISTS")
|
ifNotExistsCount := strings.Count(output, "IF NOT EXISTS")
|
||||||
if ifNotExistsCount < 3 {
|
if ifNotExistsCount < 3 {
|
||||||
@@ -1083,14 +1071,14 @@ func TestWriteAddColumnStatements(t *testing.T) {
|
|||||||
t.Logf("Generated output:\n%s", output)
|
t.Logf("Generated output:\n%s", output)
|
||||||
|
|
||||||
// Verify output contains expected elements
|
// Verify output contains expected elements
|
||||||
if !strings.Contains(output, "ALTER TABLE public.products ADD COLUMN id integer") {
|
if !strings.Contains(output, "ALTER TABLE \"public\".\"products\" ADD COLUMN IF NOT EXISTS \"id\" integer") {
|
||||||
t.Errorf("Output missing ADD COLUMN for id\nFull output:\n%s", output)
|
t.Errorf("Output missing ADD COLUMN for id\nFull output:\n%s", output)
|
||||||
}
|
}
|
||||||
if !strings.Contains(output, "ALTER TABLE public.products ADD COLUMN description text") {
|
if !strings.Contains(output, "ALTER TABLE \"public\".\"products\" ADD COLUMN IF NOT EXISTS \"description\" text") {
|
||||||
t.Errorf("Output missing ADD COLUMN for description\nFull output:\n%s", output)
|
t.Errorf("Output missing ADD COLUMN for description\nFull output:\n%s", output)
|
||||||
}
|
}
|
||||||
if !strings.Contains(output, "DO $$") {
|
if strings.Contains(output, "information_schema.columns") {
|
||||||
t.Errorf("Output missing DO block\nFull output:\n%s", output)
|
t.Errorf("column additions should use native IF NOT EXISTS\nFull output:\n%s", output)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1127,16 +1115,16 @@ func TestWriteSchema_EmitsGuardedAlterColumnTypeStatements(t *testing.T) {
|
|||||||
if !strings.Contains(output, "pg_catalog.format_type") {
|
if !strings.Contains(output, "pg_catalog.format_type") {
|
||||||
t.Fatalf("expected guarded live-type check, got:\n%s", output)
|
t.Fatalf("expected guarded live-type check, got:\n%s", output)
|
||||||
}
|
}
|
||||||
if !strings.Contains(output, "ALTER COLUMN name TYPE character varying(255)") {
|
if !strings.Contains(output, "ALTER COLUMN \"name\" TYPE character varying(255)") {
|
||||||
t.Fatalf("expected guarded alter for character varying(255), got:\n%s", output)
|
t.Fatalf("expected guarded alter for character varying(255), got:\n%s", output)
|
||||||
}
|
}
|
||||||
if !strings.Contains(output, "ARRAY['varchar(255)', 'character varying(255)']") {
|
if !strings.Contains(output, "ARRAY['varchar(255)', 'character varying(255)']") {
|
||||||
t.Fatalf("expected equivalent type spellings for varchar guard, got:\n%s", output)
|
t.Fatalf("expected equivalent type spellings for varchar guard, got:\n%s", output)
|
||||||
}
|
}
|
||||||
if !strings.Contains(output, "ALTER COLUMN tags TYPE text[]") {
|
if !strings.Contains(output, "ALTER COLUMN \"tags\" TYPE text[]") {
|
||||||
t.Fatalf("expected guarded alter for array type, got:\n%s", output)
|
t.Fatalf("expected guarded alter for array type, got:\n%s", output)
|
||||||
}
|
}
|
||||||
if !strings.Contains(output, `ALTER COLUMN tags TYPE text[] USING tags::text[];`) {
|
if !strings.Contains(output, `ALTER COLUMN "tags" TYPE text[] USING "tags"::text[];`) {
|
||||||
t.Fatalf("expected guarded alter for array type to include USING cast, got:\n%s", output)
|
t.Fatalf("expected guarded alter for array type to include USING cast, got:\n%s", output)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -1170,7 +1158,7 @@ func TestWriteSchema_EmitsGuardedAlterColumnDefaultStatements(t *testing.T) {
|
|||||||
if !strings.Contains(output, "pg_get_expr(d.adbin, d.adrelid)") {
|
if !strings.Contains(output, "pg_get_expr(d.adbin, d.adrelid)") {
|
||||||
t.Fatalf("expected guarded live-default check, got:\n%s", output)
|
t.Fatalf("expected guarded live-default check, got:\n%s", output)
|
||||||
}
|
}
|
||||||
if !strings.Contains(output, "ALTER COLUMN status SET DEFAULT 'active'") {
|
if !strings.Contains(output, "ALTER COLUMN \"status\" SET DEFAULT 'active'") {
|
||||||
t.Fatalf("expected guarded SET DEFAULT for status column, got:\n%s", output)
|
t.Fatalf("expected guarded SET DEFAULT for status column, got:\n%s", output)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -1201,12 +1189,12 @@ func TestWriteSchema_AlterColumnDefaultStripsBackticksFromFunctionExpression(t *
|
|||||||
if strings.Contains(output, "`") {
|
if strings.Contains(output, "`") {
|
||||||
t.Fatalf("expected no backticks in generated SQL, got:\n%s", output)
|
t.Fatalf("expected no backticks in generated SQL, got:\n%s", output)
|
||||||
}
|
}
|
||||||
if !strings.Contains(output, "ALTER COLUMN updatedat SET DEFAULT now()") {
|
if !strings.Contains(output, "ALTER COLUMN \"updatedat\" SET DEFAULT now()") {
|
||||||
t.Fatalf("expected guarded SET DEFAULT now() without backticks, got:\n%s", output)
|
t.Fatalf("expected guarded SET DEFAULT now() without backticks, got:\n%s", output)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestWriteSchema_GuardedAlterColumnTypeFallsBackOnConversionFailure(t *testing.T) {
|
func TestWriteSchema_GuardedAlterColumnTypePreservesConversionFailure(t *testing.T) {
|
||||||
db := models.InitDatabase("testdb")
|
db := models.InitDatabase("testdb")
|
||||||
schema := models.InitSchema("public")
|
schema := models.InitSchema("public")
|
||||||
|
|
||||||
@@ -1228,17 +1216,13 @@ func TestWriteSchema_GuardedAlterColumnTypeFallsBackOnConversionFailure(t *testi
|
|||||||
}
|
}
|
||||||
|
|
||||||
output := buf.String()
|
output := buf.String()
|
||||||
if !strings.Contains(output, "EXCEPTION WHEN OTHERS THEN") {
|
if !strings.Contains(output, "ALTER COLUMN \"name\" TYPE integer USING \"name\"::integer;") {
|
||||||
t.Fatalf("expected guarded alter to fall back on conversion failure, got:\n%s", output)
|
t.Fatalf("expected explicit type conversion, got:\n%s", output)
|
||||||
}
|
}
|
||||||
if !strings.Contains(output, "renamed_column := 'name_' || trim(both '_' from regexp_replace(lower(current_type)") {
|
for _, unsafe := range []string{"EXCEPTION WHEN OTHERS", "renamed_column", "RENAME COLUMN"} {
|
||||||
t.Fatalf("expected fallback to derive a renamed column name from the live type, got:\n%s", output)
|
if strings.Contains(output, unsafe) {
|
||||||
|
t.Fatalf("type conversion must propagate errors without replacing the column (%s):\n%s", unsafe, output)
|
||||||
}
|
}
|
||||||
if !strings.Contains(output, "RENAME COLUMN name TO %I") {
|
|
||||||
t.Fatalf("expected fallback to rename the existing column, got:\n%s", output)
|
|
||||||
}
|
|
||||||
if !strings.Contains(output, "ADD COLUMN name integer") {
|
|
||||||
t.Fatalf("expected fallback to add a fresh column with the new type, got:\n%s", output)
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1274,7 +1258,7 @@ func TestWriteSchema_EmitsGuardedAlterColumnNullabilityStatements(t *testing.T)
|
|||||||
if !strings.Contains(output, "current_not_null IS DISTINCT FROM false") {
|
if !strings.Contains(output, "current_not_null IS DISTINCT FROM false") {
|
||||||
t.Fatalf("expected guard comparing live nullability against desired value, got:\n%s", output)
|
t.Fatalf("expected guard comparing live nullability against desired value, got:\n%s", output)
|
||||||
}
|
}
|
||||||
if !strings.Contains(output, "ALTER COLUMN rid_service_instance_type DROP NOT NULL") {
|
if !strings.Contains(output, "ALTER COLUMN \"rid_service_instance_type\" DROP NOT NULL") {
|
||||||
t.Fatalf("expected guarded DROP NOT NULL for nullable column, got:\n%s", output)
|
t.Fatalf("expected guarded DROP NOT NULL for nullable column, got:\n%s", output)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -1300,13 +1284,13 @@ func TestWriteSchema_UsesStorageTypeForSerialAlterStatements(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
output := buf.String()
|
output := buf.String()
|
||||||
if !strings.Contains(output, "ALTER COLUMN id TYPE bigint") {
|
if !strings.Contains(output, "ALTER COLUMN \"id\" TYPE bigint") {
|
||||||
t.Fatalf("expected serial alter to use bigint storage type, got:\n%s", output)
|
t.Fatalf("expected serial alter to use bigint storage type, got:\n%s", output)
|
||||||
}
|
}
|
||||||
if strings.Contains(output, "ALTER COLUMN id TYPE bigserial;") {
|
if strings.Contains(output, "ALTER COLUMN \"id\" TYPE bigserial;") {
|
||||||
t.Fatalf("did not expect invalid bigserial alter statement, got:\n%s", output)
|
t.Fatalf("did not expect invalid bigserial alter statement, got:\n%s", output)
|
||||||
}
|
}
|
||||||
if !strings.Contains(output, `ALTER COLUMN id TYPE bigint USING id::bigint;`) {
|
if !strings.Contains(output, `ALTER COLUMN "id" TYPE bigint USING "id"::bigint;`) {
|
||||||
t.Fatalf("expected serial alter to include USING cast, got:\n%s", output)
|
t.Fatalf("expected serial alter to include USING cast, got:\n%s", output)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -1377,7 +1361,7 @@ func TestWriteDatabase_VectorAndPostGISColumnsCreateExtensions(t *testing.T) {
|
|||||||
func TestWriteDatabase_HNSWIndexUsesDefaultVectorOperatorClass(t *testing.T) {
|
func TestWriteDatabase_HNSWIndexUsesDefaultVectorOperatorClass(t *testing.T) {
|
||||||
output := writeDatabaseOutput(t, buildVectorSpatialSchema("hnsw", ""))
|
output := writeDatabaseOutput(t, buildVectorSpatialSchema("hnsw", ""))
|
||||||
|
|
||||||
if !strings.Contains(output, "USING hnsw (embedding vector_cosine_ops)") {
|
if !strings.Contains(output, "USING hnsw (\"embedding\" vector_cosine_ops)") {
|
||||||
t.Fatalf("expected hnsw index with default vector operator class, got:\n%s", output)
|
t.Fatalf("expected hnsw index with default vector operator class, got:\n%s", output)
|
||||||
}
|
}
|
||||||
if !strings.Contains(output, "CREATE EXTENSION IF NOT EXISTS vector;") {
|
if !strings.Contains(output, "CREATE EXTENSION IF NOT EXISTS vector;") {
|
||||||
@@ -1388,7 +1372,7 @@ func TestWriteDatabase_HNSWIndexUsesDefaultVectorOperatorClass(t *testing.T) {
|
|||||||
func TestWriteDatabase_VectorIndexHonoursRequestedOperatorClassAndStorageParameters(t *testing.T) {
|
func TestWriteDatabase_VectorIndexHonoursRequestedOperatorClassAndStorageParameters(t *testing.T) {
|
||||||
output := writeDatabaseOutput(t, buildVectorSpatialSchema("ivfflat", "opclass=vector_l2_ops; with (lists=100)"))
|
output := writeDatabaseOutput(t, buildVectorSpatialSchema("ivfflat", "opclass=vector_l2_ops; with (lists=100)"))
|
||||||
|
|
||||||
if !strings.Contains(output, "USING ivfflat (embedding vector_l2_ops) WITH (lists = 100)") {
|
if !strings.Contains(output, "USING ivfflat (\"embedding\" vector_l2_ops) WITH (lists = 100)") {
|
||||||
t.Fatalf("expected ivfflat index with requested opclass and storage parameters, got:\n%s", output)
|
t.Fatalf("expected ivfflat index with requested opclass and storage parameters, got:\n%s", output)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -1396,7 +1380,7 @@ func TestWriteDatabase_VectorIndexHonoursRequestedOperatorClassAndStorageParamet
|
|||||||
func TestWriteDatabase_VectorIndexIgnoresIncompatibleOperatorClass(t *testing.T) {
|
func TestWriteDatabase_VectorIndexIgnoresIncompatibleOperatorClass(t *testing.T) {
|
||||||
output := writeDatabaseOutput(t, buildVectorSpatialSchema("hnsw", "opclass=halfvec_l2_ops"))
|
output := writeDatabaseOutput(t, buildVectorSpatialSchema("hnsw", "opclass=halfvec_l2_ops"))
|
||||||
|
|
||||||
if !strings.Contains(output, "USING hnsw (embedding vector_cosine_ops)") {
|
if !strings.Contains(output, "USING hnsw (\"embedding\" vector_cosine_ops)") {
|
||||||
t.Fatalf("expected halfvec operator class to be rejected for a vector column, got:\n%s", output)
|
t.Fatalf("expected halfvec operator class to be rejected for a vector column, got:\n%s", output)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -1432,7 +1416,7 @@ func TestWriteDatabase_GistIndexOnGeometryUsesDefaultOperatorClass(t *testing.T)
|
|||||||
|
|
||||||
output := writeDatabaseOutput(t, db)
|
output := writeDatabaseOutput(t, db)
|
||||||
|
|
||||||
if !strings.Contains(output, "USING gist (geom)") {
|
if !strings.Contains(output, "USING gist (\"geom\")") {
|
||||||
t.Fatalf("expected gist index to rely on the PostGIS default operator class, got:\n%s", output)
|
t.Fatalf("expected gist index to rely on the PostGIS default operator class, got:\n%s", output)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -1458,7 +1442,7 @@ func TestWriteDatabase_GistIndexHonoursRequestedSpatialOperatorClass(t *testing.
|
|||||||
|
|
||||||
output := writeDatabaseOutput(t, db)
|
output := writeDatabaseOutput(t, db)
|
||||||
|
|
||||||
if !strings.Contains(output, "USING gist (geom gist_geometry_ops_nd)") {
|
if !strings.Contains(output, "USING gist (\"geom\" gist_geometry_ops_nd)") {
|
||||||
t.Fatalf("expected requested spatial operator class, got:\n%s", output)
|
t.Fatalf("expected requested spatial operator class, got:\n%s", output)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -1476,7 +1460,7 @@ func TestGenerateDatabaseStatements_VectorIndexIncludesOperatorClassAndParameter
|
|||||||
for _, want := range []string{
|
for _, want := range []string{
|
||||||
"CREATE EXTENSION IF NOT EXISTS vector",
|
"CREATE EXTENSION IF NOT EXISTS vector",
|
||||||
"CREATE EXTENSION IF NOT EXISTS postgis",
|
"CREATE EXTENSION IF NOT EXISTS postgis",
|
||||||
"USING hnsw (embedding vector_ip_ops) WITH (m = 16)",
|
"USING hnsw (\"embedding\" vector_ip_ops) WITH (m = 16)",
|
||||||
} {
|
} {
|
||||||
if !strings.Contains(joined, want) {
|
if !strings.Contains(joined, want) {
|
||||||
t.Fatalf("expected statements to contain %q, got:\n%s", want, joined)
|
t.Fatalf("expected statements to contain %q, got:\n%s", want, joined)
|
||||||
@@ -1525,7 +1509,7 @@ func TestGenerateColumnDefinition_GeneratedColumnEmitsGeneratedClauseNotDefault(
|
|||||||
|
|
||||||
got := w.generateColumnDefinition(col)
|
got := w.generateColumnDefinition(col)
|
||||||
|
|
||||||
want := "geom text GENERATED ALWAYS AS (st_makepoint(lon, lat)) STORED NOT NULL"
|
want := "\"geom\" text GENERATED ALWAYS AS (st_makepoint(lon, lat)) STORED NOT NULL"
|
||||||
if got != want {
|
if got != want {
|
||||||
t.Fatalf("generateColumnDefinition() = %q, want %q", got, want)
|
t.Fatalf("generateColumnDefinition() = %q, want %q", got, want)
|
||||||
}
|
}
|
||||||
@@ -1544,7 +1528,7 @@ func TestGenerateColumnDefinition_IdentityColumnEmitsIdentityClauseNotDefault(t
|
|||||||
|
|
||||||
got := w.generateColumnDefinition(col)
|
got := w.generateColumnDefinition(col)
|
||||||
|
|
||||||
want := "id bigint GENERATED ALWAYS AS IDENTITY NOT NULL"
|
want := "\"id\" bigint GENERATED ALWAYS AS IDENTITY NOT NULL"
|
||||||
if got != want {
|
if got != want {
|
||||||
t.Fatalf("generateColumnDefinition() = %q, want %q", got, want)
|
t.Fatalf("generateColumnDefinition() = %q, want %q", got, want)
|
||||||
}
|
}
|
||||||
@@ -1591,7 +1575,7 @@ func TestWriteDatabase_MultipleForeignKeysToSameTable(t *testing.T) {
|
|||||||
}
|
}
|
||||||
output := buf.String()
|
output := buf.String()
|
||||||
for _, name := range []string{"rid_department", "rid_manager"} {
|
for _, name := range []string{"rid_department", "rid_manager"} {
|
||||||
want := "ADD CONSTRAINT fk_employee_" + name + "\n FOREIGN KEY (" + name + ")"
|
want := "ADD CONSTRAINT " + quoteIdent("fk_employee_"+name) + "\n FOREIGN KEY (" + quoteIdent(name) + ")"
|
||||||
if !strings.Contains(output, want) {
|
if !strings.Contains(output, want) {
|
||||||
t.Fatalf("run %d: missing %q in output:\n%s", i, want, output)
|
t.Fatalf("run %d: missing %q in output:\n%s", i, want, output)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -186,11 +186,30 @@ Future support planned for:
|
|||||||
- SQLite
|
- SQLite
|
||||||
- Generic SQL via database/sql
|
- Generic SQL via database/sql
|
||||||
|
|
||||||
## Transaction Behavior
|
## Execution and transaction behavior
|
||||||
|
|
||||||
**Current**: Each script executes in its own implicit transaction (PostgreSQL default behavior)
|
This is a raw PostgreSQL script runner, not a migration manager. It has no applied
|
||||||
|
history, checksums, migration lock, or duplicate-run protection. Re-running repeats
|
||||||
|
the supplied scripts. Scripts may contain transaction control, so RelSpec does not
|
||||||
|
wrap them in a transaction and rejects `WriterOptions.AtomicExecution`.
|
||||||
|
|
||||||
**Future Enhancement**: Option to wrap all scripts in a single transaction for atomic execution with rollback on error.
|
By default execution stops on the first SQL or embed-preparation failure. Set
|
||||||
|
`WriterOptions.ContinueOnError`, legacy `Metadata["ignore_errors"]`, or CLI
|
||||||
|
`--ignore-errors` / `--continue-on-error` / `--coe` to attempt later scripts. Any
|
||||||
|
failure still returns a nonzero result and prevents dependent jobs from running.
|
||||||
|
|
||||||
|
The shared `writers.ExecutionReport` is available at `Metadata["execution_report"]`;
|
||||||
|
`Metadata["report_path"]` requests a JSON report and report-write failures are errors.
|
||||||
|
Counters and operations cover every schema in the invocation, including scripts
|
||||||
|
skipped after stopping or because they are empty. Legacy `execution_total`,
|
||||||
|
`execution_success`, `execution_failed`, `execution_skipped`, and
|
||||||
|
`execution_rolled_back` metadata counters are also populated on failure.
|
||||||
|
|
||||||
|
Reports use `outcome: script_managed` and successful script status `executed`, rather
|
||||||
|
than claiming committed database changes. A script may commit, roll back, or leave a
|
||||||
|
transaction open, and a failed script may already have committed work. The report
|
||||||
|
records which scripts succeeded/failed/skipped; it cannot infer statement-level
|
||||||
|
commit or rollback inside arbitrary scripts. The connection closes after execution.
|
||||||
|
|
||||||
## Performance Considerations
|
## Performance Considerations
|
||||||
|
|
||||||
@@ -226,7 +245,7 @@ docker run -d --name postgres-test \
|
|||||||
postgres:16
|
postgres:16
|
||||||
|
|
||||||
# Run your integration tests
|
# Run your integration tests
|
||||||
go test -tags=integration ./pkg/writers/sqlexec/
|
RELSPEC_TEST_PG_CONN='postgres://postgres:test@127.0.0.1:5432/testdb?sslmode=disable' go test -count=1 ./pkg/writers/sqlexec/
|
||||||
|
|
||||||
# Cleanup
|
# Cleanup
|
||||||
docker stop postgres-test
|
docker stop postgres-test
|
||||||
@@ -237,7 +256,7 @@ docker rm postgres-test
|
|||||||
|
|
||||||
- `WriteTable()` is not supported (returns error)
|
- `WriteTable()` is not supported (returns error)
|
||||||
- Requires PostgreSQL connection (no offline mode)
|
- Requires PostgreSQL connection (no offline mode)
|
||||||
- No built-in transaction wrapping (yet)
|
- No executor-level transaction wrapping for raw scripts
|
||||||
- No rollback script support (yet, though `models.Script.Rollback` field exists)
|
- No rollback script support (yet, though `models.Script.Rollback` field exists)
|
||||||
|
|
||||||
## Related
|
## Related
|
||||||
@@ -245,3 +264,11 @@ docker rm postgres-test
|
|||||||
- **SQL Directory Reader**: `pkg/readers/sqldir/` - Read scripts from filesystem
|
- **SQL Directory Reader**: `pkg/readers/sqldir/` - Read scripts from filesystem
|
||||||
- **Script Model**: `pkg/models/models.go` - Script structure definition
|
- **Script Model**: `pkg/models/models.go` - Script structure definition
|
||||||
- **pgx Documentation**: https://github.com/jackc/pgx - PostgreSQL driver docs
|
- **pgx Documentation**: https://github.com/jackc/pgx - PostgreSQL driver docs
|
||||||
|
|
||||||
|
## Model embed access
|
||||||
|
|
||||||
|
SQL directory readers expand embed directives within their trusted input root.
|
||||||
|
When executing unexpanded directives from a model, explicitly configure
|
||||||
|
`WriterOptions.Metadata["asset_root"]` with the allowed directory. Script
|
||||||
|
`source_path` metadata alone never authorizes file reads. Paths, symlinks, regular
|
||||||
|
file types, and the 64 MiB asset-size limit are checked before reading.
|
||||||
|
|||||||
+55
-109
@@ -42,22 +42,25 @@ func (w *Writer) WriteDatabase(db *models.Database) error {
|
|||||||
return fmt.Errorf("connection_string is required in writer metadata")
|
return fmt.Errorf("connection_string is required in writer metadata")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Preserve schema order and script ordering within each schema, but aggregate once.
|
||||||
|
var scripts []*models.Script
|
||||||
|
for _, schema := range db.Schemas {
|
||||||
|
for _, script := range sortedScripts(schema.Scripts) {
|
||||||
|
scriptCopy := *script
|
||||||
|
scriptCopy.Name = schema.Name + "." + script.Name
|
||||||
|
scripts = append(scripts, &scriptCopy)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// Connect to database
|
// Connect to database
|
||||||
ctx := context.Background()
|
ctx := context.Background()
|
||||||
conn, err := pgsql.Connect(ctx, connString, "writer-sqlexec")
|
conn, err := pgsql.Connect(ctx, connString, "writer-sqlexec")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("failed to connect to database: %w", err)
|
return writers.FailExecution(w.options, "PostgreSQL scripts", scriptOperations(scripts), fmt.Errorf("failed to connect to database: %w", err))
|
||||||
}
|
}
|
||||||
defer conn.Close(ctx)
|
defer conn.Close(ctx)
|
||||||
|
|
||||||
// Execute scripts from all schemas
|
return w.runScripts(ctx, conn, scripts)
|
||||||
for _, schema := range db.Schemas {
|
|
||||||
if err := w.executeScripts(ctx, conn, schema.Scripts); err != nil {
|
|
||||||
return fmt.Errorf("failed to execute scripts from schema %s: %w", schema.Name, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// WriteSchema executes all scripts from a single schema
|
// WriteSchema executes all scripts from a single schema
|
||||||
@@ -76,7 +79,7 @@ func (w *Writer) WriteSchema(schema *models.Schema) error {
|
|||||||
ctx := context.Background()
|
ctx := context.Background()
|
||||||
conn, err := pgsql.Connect(ctx, connString, "writer-sqlexec")
|
conn, err := pgsql.Connect(ctx, connString, "writer-sqlexec")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("failed to connect to database: %w", err)
|
return writers.FailExecution(w.options, "PostgreSQL scripts", scriptOperations(sortedScripts(schema.Scripts)), fmt.Errorf("failed to connect to database: %w", err))
|
||||||
}
|
}
|
||||||
defer conn.Close(ctx)
|
defer conn.Close(ctx)
|
||||||
|
|
||||||
@@ -93,119 +96,62 @@ func (w *Writer) WriteTable(table *models.Table) error {
|
|||||||
return fmt.Errorf("WriteTable is not supported for SQL script execution")
|
return fmt.Errorf("WriteTable is not supported for SQL script execution")
|
||||||
}
|
}
|
||||||
|
|
||||||
// executeScripts executes scripts in Priority, Sequence, then Name order
|
// executeScripts executes scripts in Priority, Sequence, then Name order.
|
||||||
func (w *Writer) executeScripts(ctx context.Context, conn *pgx.Conn, scripts []*models.Script) error {
|
func (w *Writer) executeScripts(ctx context.Context, conn *pgx.Conn, scripts []*models.Script) error {
|
||||||
if len(scripts) == 0 {
|
return w.runScripts(ctx, conn, sortedScripts(scripts))
|
||||||
return nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Check if we should ignore errors
|
func sortedScripts(scripts []*models.Script) []*models.Script {
|
||||||
ignoreErrors := false
|
ordered := append([]*models.Script(nil), scripts...)
|
||||||
if val, ok := w.options.Metadata["ignore_errors"].(bool); ok {
|
sort.SliceStable(ordered, func(i, j int) bool {
|
||||||
ignoreErrors = val
|
if ordered[i].Priority != ordered[j].Priority {
|
||||||
|
return ordered[i].Priority < ordered[j].Priority
|
||||||
}
|
}
|
||||||
|
if ordered[i].Sequence != ordered[j].Sequence {
|
||||||
// Track failed scripts and execution counts
|
return ordered[i].Sequence < ordered[j].Sequence
|
||||||
var failedScripts []struct {
|
|
||||||
name string
|
|
||||||
priority int
|
|
||||||
sequence uint
|
|
||||||
err error
|
|
||||||
}
|
}
|
||||||
successCount := 0
|
return ordered[i].Name < ordered[j].Name
|
||||||
totalCount := 0
|
|
||||||
|
|
||||||
// Sort scripts by Priority (ascending), Sequence (ascending), then Name (ascending)
|
|
||||||
sortedScripts := make([]*models.Script, len(scripts))
|
|
||||||
copy(sortedScripts, scripts)
|
|
||||||
sort.Slice(sortedScripts, func(i, j int) bool {
|
|
||||||
if sortedScripts[i].Priority != sortedScripts[j].Priority {
|
|
||||||
return sortedScripts[i].Priority < sortedScripts[j].Priority
|
|
||||||
}
|
|
||||||
if sortedScripts[i].Sequence != sortedScripts[j].Sequence {
|
|
||||||
return sortedScripts[i].Sequence < sortedScripts[j].Sequence
|
|
||||||
}
|
|
||||||
return sortedScripts[i].Name < sortedScripts[j].Name
|
|
||||||
})
|
})
|
||||||
|
return ordered
|
||||||
// Execute each script in order
|
|
||||||
for _, script := range sortedScripts {
|
|
||||||
if script.SQL == "" {
|
|
||||||
continue
|
|
||||||
}
|
}
|
||||||
|
|
||||||
totalCount++
|
func (w *Writer) runScripts(ctx context.Context, conn *pgx.Conn, scripts []*models.Script) error {
|
||||||
fmt.Printf("Executing script: %s (Priority=%d, Sequence=%d)\n",
|
operations := scriptOperations(scripts)
|
||||||
script.Name, script.Priority, script.Sequence)
|
backend := writers.ExecutionBackend{ScriptManaged: true, Exec: func(ctx context.Context, i int, _ string) error {
|
||||||
|
script := scripts[i]
|
||||||
// Execute the SQL script
|
fmt.Printf("Executing script: %s (Priority=%d, Sequence=%d)\n", script.Name, script.Priority, script.Sequence)
|
||||||
sql, err := processEmbedDirectives(script)
|
assetRoot, _ := w.options.Metadata["asset_root"].(string)
|
||||||
|
sql, err := processEmbedDirectives(script, assetRoot)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
if ignoreErrors {
|
return fmt.Errorf("preparing script %s: %w", script.Name, err)
|
||||||
fmt.Printf("⚠ Error preparing %s: %v (continuing due to --ignore-errors)\n", script.Name, err)
|
|
||||||
failedScripts = append(failedScripts, struct {
|
|
||||||
name string
|
|
||||||
priority int
|
|
||||||
sequence uint
|
|
||||||
err error
|
|
||||||
}{
|
|
||||||
name: script.Name,
|
|
||||||
priority: script.Priority,
|
|
||||||
sequence: script.Sequence,
|
|
||||||
err: err,
|
|
||||||
})
|
|
||||||
continue
|
|
||||||
}
|
}
|
||||||
return fmt.Errorf("script %s (Priority=%d, Sequence=%d): %w",
|
if _, err := conn.Exec(ctx, sql); err != nil {
|
||||||
script.Name, script.Priority, script.Sequence, err)
|
return fmt.Errorf("script %s: %w", script.Name, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
_, err = conn.Exec(ctx, sql)
|
|
||||||
if err != nil {
|
|
||||||
if ignoreErrors {
|
|
||||||
fmt.Printf("⚠ Error executing %s: %v (continuing due to --ignore-errors)\n", script.Name, err)
|
|
||||||
failedScripts = append(failedScripts, struct {
|
|
||||||
name string
|
|
||||||
priority int
|
|
||||||
sequence uint
|
|
||||||
err error
|
|
||||||
}{
|
|
||||||
name: script.Name,
|
|
||||||
priority: script.Priority,
|
|
||||||
sequence: script.Sequence,
|
|
||||||
err: err,
|
|
||||||
})
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
return fmt.Errorf("script %s (Priority=%d, Sequence=%d): %w",
|
|
||||||
script.Name, script.Priority, script.Sequence, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
successCount++
|
|
||||||
fmt.Printf("✓ Successfully executed: %s\n", script.Name)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Store execution results in metadata for caller
|
|
||||||
w.options.Metadata["execution_total"] = totalCount
|
|
||||||
w.options.Metadata["execution_success"] = successCount
|
|
||||||
w.options.Metadata["execution_failed"] = len(failedScripts)
|
|
||||||
|
|
||||||
// Print summary of failed scripts if any
|
|
||||||
if len(failedScripts) > 0 {
|
|
||||||
fmt.Printf("\n⚠ Failed Scripts Summary (%d failed):\n", len(failedScripts))
|
|
||||||
for i, failed := range failedScripts {
|
|
||||||
fmt.Printf(" %d. %s (Priority=%d, Sequence=%d)\n Error: %v\n",
|
|
||||||
i+1, failed.name, failed.priority, failed.sequence, failed.err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
|
}}
|
||||||
|
// Supplied scripts can manage transactions themselves. Do not promise whole-script rollback.
|
||||||
|
return writers.RunExecution(ctx, w.options, &writers.ExecutionReport{Dialect: "PostgreSQL scripts"}, operations, backend)
|
||||||
}
|
}
|
||||||
|
|
||||||
func processEmbedDirectives(script *models.Script) (string, error) {
|
func scriptOperations(scripts []*models.Script) []writers.ExecutionOperation {
|
||||||
sqlPath, _ := script.Metadata[assetloader.ScriptSourcePathMetadataKey].(string)
|
operations := make([]writers.ExecutionOperation, len(scripts))
|
||||||
if sqlPath == "" {
|
for i, script := range scripts {
|
||||||
|
operations[i] = writers.ExecutionOperation{Name: script.Name, Statement: script.SQL}
|
||||||
|
}
|
||||||
|
return operations
|
||||||
|
}
|
||||||
|
|
||||||
|
func processEmbedDirectives(script *models.Script, assetRoot string) (string, error) {
|
||||||
|
if !assetloader.HasEmbedDirectives(script.SQL) {
|
||||||
return script.SQL, nil
|
return script.SQL, nil
|
||||||
}
|
}
|
||||||
return assetloader.ProcessEmbedDirectives(sqlPath, script.SQL)
|
if assetRoot == "" {
|
||||||
|
return "", fmt.Errorf("asset_root is required to resolve embed directives from a model; source_path metadata does not authorize file access")
|
||||||
|
}
|
||||||
|
sqlPath, _ := script.Metadata[assetloader.ScriptSourcePathMetadataKey].(string)
|
||||||
|
if sqlPath == "" {
|
||||||
|
return "", fmt.Errorf("source_path is required for model embed directives")
|
||||||
|
}
|
||||||
|
return assetloader.ProcessEmbedDirectivesWithOptions(sqlPath, script.SQL, assetloader.EmbedOptions{AssetRoot: assetRoot})
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -2,6 +2,7 @@ package sqlexec
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
@@ -121,7 +122,7 @@ func TestLive_ExecuteScriptsOrder(t *testing.T) {
|
|||||||
if strings.Join(got, ",") != strings.Join(want, ",") {
|
if strings.Join(got, ",") != strings.Join(want, ",") {
|
||||||
t.Errorf("execution order = %v, want %v", got, want)
|
t.Errorf("execution order = %v, want %v", got, want)
|
||||||
}
|
}
|
||||||
if opts.Metadata["execution_total"] != 6 || opts.Metadata["execution_success"] != 6 || opts.Metadata["execution_failed"] != 0 {
|
if opts.Metadata["execution_total"] != 7 || opts.Metadata["execution_success"] != 6 || opts.Metadata["execution_failed"] != 0 {
|
||||||
t.Errorf("counts: %v", opts.Metadata)
|
t.Errorf("counts: %v", opts.Metadata)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -160,8 +161,8 @@ func TestLive_IgnoreErrorsContinues(t *testing.T) {
|
|||||||
{Name: "02_ok", Priority: 2, SQL: fmt.Sprintf("CREATE TABLE %s.after(id int);", schema)},
|
{Name: "02_ok", Priority: 2, SQL: fmt.Sprintf("CREATE TABLE %s.after(id int);", schema)},
|
||||||
}
|
}
|
||||||
opts := liveOptions(connString, map[string]interface{}{"ignore_errors": true})
|
opts := liveOptions(connString, map[string]interface{}{"ignore_errors": true})
|
||||||
if err := NewWriter(opts).WriteSchema(&models.Schema{Name: schema, Scripts: scripts}); err != nil {
|
if err := NewWriter(opts).WriteSchema(&models.Schema{Name: schema, Scripts: scripts}); err == nil {
|
||||||
t.Fatalf("ignore_errors must not fail: %v", err)
|
t.Fatal("continuation must still return failure")
|
||||||
}
|
}
|
||||||
if opts.Metadata["execution_total"] != 2 || opts.Metadata["execution_success"] != 1 || opts.Metadata["execution_failed"] != 1 {
|
if opts.Metadata["execution_total"] != 2 || opts.Metadata["execution_success"] != 1 || opts.Metadata["execution_failed"] != 1 {
|
||||||
t.Errorf("counts: %v", opts.Metadata)
|
t.Errorf("counts: %v", opts.Metadata)
|
||||||
@@ -186,8 +187,8 @@ func TestLive_EmbedDirectiveErrorHandling(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
opts := liveOptions(connString, map[string]interface{}{"ignore_errors": true})
|
opts := liveOptions(connString, map[string]interface{}{"ignore_errors": true})
|
||||||
if err := NewWriter(opts).WriteSchema(&models.Schema{Name: schema, Scripts: []*models.Script{bad}}); err != nil {
|
if err := NewWriter(opts).WriteSchema(&models.Schema{Name: schema, Scripts: []*models.Script{bad}}); err == nil {
|
||||||
t.Errorf("ignore_errors: %v", err)
|
t.Error("embed preparation failures must propagate even when continuing")
|
||||||
}
|
}
|
||||||
if opts.Metadata["execution_failed"] != 1 {
|
if opts.Metadata["execution_failed"] != 1 {
|
||||||
t.Errorf("counts: %v", opts.Metadata)
|
t.Errorf("counts: %v", opts.Metadata)
|
||||||
@@ -217,7 +218,29 @@ func TestLive_WriteDatabaseMultiSchema(t *testing.T) {
|
|||||||
// A failure in one schema aborts and names that schema.
|
// A failure in one schema aborts and names that schema.
|
||||||
db.Schemas[1].Scripts[0].SQL = "SELECT * FROM definitely_missing_table;"
|
db.Schemas[1].Scripts[0].SQL = "SELECT * FROM definitely_missing_table;"
|
||||||
err := NewWriter(liveOptions(connString, nil)).WriteDatabase(db)
|
err := NewWriter(liveOptions(connString, nil)).WriteDatabase(db)
|
||||||
if err == nil || !strings.Contains(err.Error(), "schema "+s2) {
|
if err == nil || !strings.Contains(err.Error(), s2+".b") {
|
||||||
t.Errorf("expected error naming schema %s, got %v", s2, err)
|
t.Errorf("expected error naming schema %s, got %v", s2, err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestLive_MultiSchemaFailuresAggregate(t *testing.T) {
|
||||||
|
connection := liveConn(t)
|
||||||
|
opts := liveOptions(connection, nil)
|
||||||
|
opts.ContinueOnError = true
|
||||||
|
db := &models.Database{Schemas: []*models.Schema{
|
||||||
|
{Name: "one", Scripts: []*models.Script{{Name: "bad", SQL: "SELECT 1 / 0"}, {Name: "empty"}}},
|
||||||
|
{Name: "two", Scripts: []*models.Script{{Name: "ok", SQL: "SELECT 1"}}},
|
||||||
|
}}
|
||||||
|
err := NewWriter(opts).WriteDatabase(db)
|
||||||
|
var failure *writers.ExecutionFailure
|
||||||
|
if !errors.As(err, &failure) {
|
||||||
|
t.Fatalf("expected structured failure: %v", err)
|
||||||
|
}
|
||||||
|
report := failure.Report
|
||||||
|
if report.TotalStatements != 3 || report.FailedStatements != 1 || report.ExecutedStatements != 1 || report.SkippedStatements != 1 || report.Outcome != "script_managed" {
|
||||||
|
t.Fatalf("incorrect aggregate: %+v", report)
|
||||||
|
}
|
||||||
|
if report.Operations[0].Name != "one.bad" || report.Operations[2].Name != "two.ok" || report.Operations[2].Status != "executed" {
|
||||||
|
t.Fatalf("script details: %+v", report.Operations)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -232,7 +232,7 @@ func TestProcessEmbedDirectives_UsesScriptSourcePath(t *testing.T) {
|
|||||||
script.SQL = "-- @embed: path=body.txt var=:body mode=text\nSELECT :body;"
|
script.SQL = "-- @embed: path=body.txt var=:body mode=text\nSELECT :body;"
|
||||||
script.Metadata[assetloader.ScriptSourcePathMetadataKey] = sqlPath
|
script.Metadata[assetloader.ScriptSourcePathMetadataKey] = sqlPath
|
||||||
|
|
||||||
got, err := processEmbedDirectives(script)
|
got, err := processEmbedDirectives(script, dir)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("processEmbedDirectives failed: %v", err)
|
t.Fatalf("processEmbedDirectives failed: %v", err)
|
||||||
}
|
}
|
||||||
@@ -241,15 +241,28 @@ func TestProcessEmbedDirectives_UsesScriptSourcePath(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestProcessEmbedDirectives_NoSourcePathLeavesSQLUnchanged(t *testing.T) {
|
func TestProcessEmbedDirectives_NoSourcePathCannotAuthorizeRead(t *testing.T) {
|
||||||
script := models.InitScript("seed")
|
script := models.InitScript("seed")
|
||||||
script.SQL = "-- @embed: path=missing.txt var=:body mode=text\nSELECT :body;"
|
script.SQL = "-- @embed: path=missing.txt var=:body mode=text\nSELECT :body;"
|
||||||
|
|
||||||
got, err := processEmbedDirectives(script)
|
got, err := processEmbedDirectives(script, t.TempDir())
|
||||||
if err != nil {
|
if err == nil || got != "" {
|
||||||
t.Fatalf("processEmbedDirectives failed: %v", err)
|
t.Fatalf("expected missing source_path failure, got %q %v", got, err)
|
||||||
}
|
}
|
||||||
if got != script.SQL {
|
}
|
||||||
t.Fatalf("expected unchanged SQL without source path, got:\n%s", got)
|
|
||||||
|
func TestProcessEmbedDirectivesModelSourcePathIsNotAuthorization(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
if err := os.WriteFile(filepath.Join(dir, "secret.txt"), []byte("secret"), 0o600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
script := models.InitScript("untrusted")
|
||||||
|
script.SQL = "-- @embed: path=secret.txt var=:body mode=text\nSELECT :body;"
|
||||||
|
script.Metadata[assetloader.ScriptSourcePathMetadataKey] = filepath.Join(dir, "fake.sql")
|
||||||
|
if _, err := processEmbedDirectives(script, ""); err == nil || !strings.Contains(err.Error(), "asset_root is required") {
|
||||||
|
t.Fatalf("source metadata must not authorize file read: %v", err)
|
||||||
|
}
|
||||||
|
if _, err := processEmbedDirectives(script, t.TempDir()); err == nil {
|
||||||
|
t.Fatal("source path outside trusted root must fail")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -117,50 +117,49 @@ func (c *statementCollector) Write(p []byte) (int, error) {
|
|||||||
// against the SQLite database file at dbPath.
|
// against the SQLite database file at dbPath.
|
||||||
func (w *Writer) executeDatabaseSQL(db *models.Database, dbPath string) error {
|
func (w *Writer) executeDatabaseSQL(db *models.Database, dbPath string) error {
|
||||||
collector := &statementCollector{}
|
collector := &statementCollector{}
|
||||||
|
saved := w.writer
|
||||||
w.writer = collector
|
w.writer = collector
|
||||||
|
defer func() { w.writer = saved }()
|
||||||
if err := w.writeContent(db); err != nil {
|
if err := w.writeContent(db); err != nil {
|
||||||
return fmt.Errorf("failed to generate SQL statements: %w", err)
|
return fmt.Errorf("failed to generate SQL statements: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
conn, err := sql.Open("sqlite", dbPath)
|
conn, err := sql.Open("sqlite", dbPath)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("failed to open sqlite database %q: %w", dbPath, err)
|
return writers.FailExecution(w.options, "SQLite", writers.StatementOperations(collector.statements), fmt.Errorf("failed to open sqlite database %q: %w", dbPath, err))
|
||||||
}
|
}
|
||||||
defer conn.Close()
|
defer conn.Close()
|
||||||
|
|
||||||
|
// Pin one connection so connection-local settings (including foreign_keys) persist.
|
||||||
ctx := context.Background()
|
ctx := context.Background()
|
||||||
ignoreErrors := false
|
session, err := conn.Conn(ctx)
|
||||||
if val, ok := w.options.Metadata["ignore_errors"].(bool); ok {
|
if err != nil {
|
||||||
ignoreErrors = val
|
return writers.FailExecution(w.options, "SQLite", writers.StatementOperations(collector.statements), err)
|
||||||
}
|
}
|
||||||
|
defer session.Close()
|
||||||
total, executed := 0, 0
|
var tx *sql.Tx
|
||||||
var execErrors []string
|
backend := writers.ExecutionBackend{
|
||||||
for _, stmt := range collector.statements {
|
Exec: func(ctx context.Context, _ int, stmt string) error {
|
||||||
if strings.HasPrefix(stmt, "--") {
|
if tx != nil {
|
||||||
continue
|
_, err := tx.ExecContext(ctx, stmt)
|
||||||
|
return err
|
||||||
}
|
}
|
||||||
|
_, err := session.ExecContext(ctx, stmt)
|
||||||
total++
|
return err
|
||||||
if _, err := conn.ExecContext(ctx, stmt); err != nil {
|
},
|
||||||
execErrors = append(execErrors, fmt.Sprintf("statement %d (%s): %v", total, truncateStatement(stmt), err))
|
Begin: func(ctx context.Context) error {
|
||||||
if !ignoreErrors {
|
// SQLite ignores foreign_keys changes inside transactions.
|
||||||
break
|
if _, err := session.ExecContext(ctx, "PRAGMA foreign_keys=ON"); err != nil {
|
||||||
|
return err
|
||||||
}
|
}
|
||||||
continue
|
var err error
|
||||||
|
tx, err = session.BeginTx(ctx, nil)
|
||||||
|
return err
|
||||||
|
},
|
||||||
|
Commit: func(context.Context) error { return tx.Commit() },
|
||||||
|
Rollback: func(context.Context) error { return tx.Rollback() },
|
||||||
}
|
}
|
||||||
executed++
|
return writers.RunExecution(ctx, w.options, &writers.ExecutionReport{Dialect: "SQLite"}, writers.StatementOperations(collector.statements), backend)
|
||||||
}
|
|
||||||
|
|
||||||
w.options.Metadata["execution_total"] = total
|
|
||||||
w.options.Metadata["execution_success"] = executed
|
|
||||||
w.options.Metadata["execution_failed"] = len(execErrors)
|
|
||||||
|
|
||||||
if len(execErrors) > 0 {
|
|
||||||
return fmt.Errorf("failed to execute %d/%d statement(s) against %q:\n%s", len(execErrors), total, dbPath, strings.Join(execErrors, "\n"))
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// truncateStatement shortens a SQL statement for error messages.
|
// truncateStatement shortens a SQL statement for error messages.
|
||||||
|
|||||||
@@ -2,6 +2,7 @@ package sqlite
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"database/sql"
|
"database/sql"
|
||||||
|
"errors"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"strings"
|
"strings"
|
||||||
@@ -182,7 +183,7 @@ func TestExecute_StopsOnErrorUnlessIgnored(t *testing.T) {
|
|||||||
path := prepare(t)
|
path := prepare(t)
|
||||||
opts := &writers.WriterOptions{Metadata: map[string]any{"connection_string": path}}
|
opts := &writers.WriterOptions{Metadata: map[string]any{"connection_string": path}}
|
||||||
err := NewWriter(opts).WriteDatabase(shopDB())
|
err := NewWriter(opts).WriteDatabase(shopDB())
|
||||||
if err == nil || !strings.Contains(err.Error(), "failed to execute") {
|
if err == nil || !strings.Contains(err.Error(), "execution failed") {
|
||||||
t.Fatalf("expected failure, got %v", err)
|
t.Fatalf("expected failure, got %v", err)
|
||||||
}
|
}
|
||||||
if opts.Metadata["execution_failed"] != 1 {
|
if opts.Metadata["execution_failed"] != 1 {
|
||||||
@@ -248,3 +249,36 @@ func TestDBMLFixtureExecutes(t *testing.T) {
|
|||||||
t.Errorf("nothing executed: %+v", opts.Metadata)
|
t.Errorf("nothing executed: %+v", opts.Metadata)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestExecuteAtomicRollback(t *testing.T) {
|
||||||
|
path := filepath.Join(t.TempDir(), "atomic.db")
|
||||||
|
opts := &writers.WriterOptions{AtomicExecution: true, Metadata: map[string]any{"connection_string": path}}
|
||||||
|
db := models.InitDatabase("atomic")
|
||||||
|
schema := models.InitSchema("main")
|
||||||
|
for _, name := range []string{"first", "bad", "last"} {
|
||||||
|
table := models.InitTable(name, "main")
|
||||||
|
col := models.InitColumn("id", name, "main")
|
||||||
|
col.Type = "integer"
|
||||||
|
table.Columns[col.Name] = col
|
||||||
|
schema.Tables = append(schema.Tables, table)
|
||||||
|
}
|
||||||
|
// A duplicate column creates an invalid table definition in the middle.
|
||||||
|
bad := schema.Tables[1]
|
||||||
|
bad.Columns["duplicate"] = models.InitColumn("id", "bad", "main")
|
||||||
|
bad.Columns["duplicate"].Type = "integer"
|
||||||
|
db.Schemas = append(db.Schemas, schema)
|
||||||
|
err := NewWriter(opts).WriteDatabase(db)
|
||||||
|
var failure *writers.ExecutionFailure
|
||||||
|
if !errors.As(err, &failure) || failure.Report.Outcome != "rolled_back" || failure.Report.RolledBackStatements == 0 || failure.Report.ExecutedStatements != 0 {
|
||||||
|
t.Fatalf("rollback report: %v", err)
|
||||||
|
}
|
||||||
|
conn, err := sql.Open("sqlite", path)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer conn.Close()
|
||||||
|
var count int
|
||||||
|
if err := conn.QueryRow("SELECT count(*) FROM sqlite_master WHERE type='table'").Scan(&count); err != nil || count != 0 {
|
||||||
|
t.Fatalf("tables survived rollback: %d %v", count, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -168,15 +168,22 @@ relspec templ --from json --from-path schema.json --template test.tmpl
|
|||||||
|
|
||||||
## Multi-file Output
|
## Multi-file Output
|
||||||
|
|
||||||
For multi-file modes, the writer:
|
For multi-file modes, `OutputPath` is always the output directory, including when
|
||||||
|
it does not yet exist or has no trailing slash. An empty path uses the current
|
||||||
|
directory. Database mode continues to treat `OutputPath` as one explicit file.
|
||||||
|
|
||||||
1. Iterates through items (schemas/scripts/tables)
|
The writer renders and validates all filenames and contents before writing. Names
|
||||||
2. Creates `TemplateData` for each item
|
must be relative, may contain safe subdirectories, and cannot contain traversal,
|
||||||
3. Executes template with item data
|
absolute/drive/UNC paths, backslashes, Windows device names, alternate streams,
|
||||||
4. Generates filename using `filename_pattern` template
|
control characters, or trailing dots/spaces. Duplicate filenames (including case
|
||||||
5. Writes output to generated filename
|
variants) and file/directory collisions fail before any artifact is replaced.
|
||||||
|
|
||||||
Output directory is created automatically if it doesn't exist.
|
Output directories are created automatically. Existing symlink roots, parents, and
|
||||||
|
files are rejected. Writes and renames use `os.Root` to retain containment if paths
|
||||||
|
change during generation. Each file is staged exclusively, closed successfully,
|
||||||
|
and renamed into place; existing permissions are preserved on replacement.
|
||||||
|
Multi-file publication is per file, so an I/O error partway through finalization can
|
||||||
|
still leave a mixed generation (the broader audit A16 staging/manifest work remains).
|
||||||
|
|
||||||
## Filename Pattern Execution
|
## Filename Pattern Execution
|
||||||
|
|
||||||
|
|||||||
+121
-93
@@ -2,6 +2,9 @@ package template
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"bytes"
|
"bytes"
|
||||||
|
"crypto/rand"
|
||||||
|
"encoding/hex"
|
||||||
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
@@ -9,6 +12,7 @@ import (
|
|||||||
"text/template"
|
"text/template"
|
||||||
|
|
||||||
"git.warky.dev/wdevs/relspecgo/pkg/models"
|
"git.warky.dev/wdevs/relspecgo/pkg/models"
|
||||||
|
"git.warky.dev/wdevs/relspecgo/pkg/safefile"
|
||||||
"git.warky.dev/wdevs/relspecgo/pkg/writers"
|
"git.warky.dev/wdevs/relspecgo/pkg/writers"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -125,97 +129,139 @@ func (w *Writer) executeDatabaseMode(db *models.Database) error {
|
|||||||
return w.writeOutput(output, w.options.OutputPath)
|
return w.writeOutput(output, w.options.OutputPath)
|
||||||
}
|
}
|
||||||
|
|
||||||
// executeSchemaMode executes the template once per schema
|
// Multi-file generation renders and validates every artifact before touching output.
|
||||||
func (w *Writer) executeSchemaMode(db *models.Database) error {
|
func (w *Writer) executeSchemaMode(db *models.Database) error {
|
||||||
|
var data []*TemplateData
|
||||||
for _, schema := range db.Schemas {
|
for _, schema := range db.Schemas {
|
||||||
data := NewSchemaData(schema, w.options.Metadata)
|
data = append(data, NewSchemaData(schema, w.options.Metadata))
|
||||||
output, err := w.executeTemplate(data)
|
}
|
||||||
if err != nil {
|
return w.executeFiles(data)
|
||||||
return fmt.Errorf("failed to execute template for schema %s: %w", schema.Name, err)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
filename, err := w.generateFilename(data)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("failed to generate filename for schema %s: %w", schema.Name, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if err := w.writeOutput(output, filename); err != nil {
|
|
||||||
return fmt.Errorf("failed to write output for schema %s: %w", schema.Name, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// executeDomainMode executes the template once per domain
|
|
||||||
func (w *Writer) executeDomainMode(db *models.Database) error {
|
func (w *Writer) executeDomainMode(db *models.Database) error {
|
||||||
|
var data []*TemplateData
|
||||||
for _, domain := range db.Domains {
|
for _, domain := range db.Domains {
|
||||||
data := NewDomainData(domain, db, w.options.Metadata)
|
data = append(data, NewDomainData(domain, db, w.options.Metadata))
|
||||||
output, err := w.executeTemplate(data)
|
}
|
||||||
if err != nil {
|
return w.executeFiles(data)
|
||||||
return fmt.Errorf("failed to execute template for domain %s: %w", domain.Name, err)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
filename, err := w.generateFilename(data)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("failed to generate filename for domain %s: %w", domain.Name, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if err := w.writeOutput(output, filename); err != nil {
|
|
||||||
return fmt.Errorf("failed to write output for domain %s: %w", domain.Name, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// executeScriptMode executes the template once per script
|
|
||||||
func (w *Writer) executeScriptMode(db *models.Database) error {
|
func (w *Writer) executeScriptMode(db *models.Database) error {
|
||||||
|
var data []*TemplateData
|
||||||
for _, schema := range db.Schemas {
|
for _, schema := range db.Schemas {
|
||||||
for _, script := range schema.Scripts {
|
for _, script := range schema.Scripts {
|
||||||
data := NewScriptData(script, schema, db, w.options.Metadata)
|
data = append(data, NewScriptData(script, schema, db, w.options.Metadata))
|
||||||
output, err := w.executeTemplate(data)
|
}
|
||||||
if err != nil {
|
}
|
||||||
return fmt.Errorf("failed to execute template for script %s: %w", script.Name, err)
|
return w.executeFiles(data)
|
||||||
}
|
}
|
||||||
|
|
||||||
filename, err := w.generateFilename(data)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("failed to generate filename for script %s: %w", script.Name, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if err := w.writeOutput(output, filename); err != nil {
|
|
||||||
return fmt.Errorf("failed to write output for script %s: %w", script.Name, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// executeTableMode executes the template once per table
|
|
||||||
func (w *Writer) executeTableMode(db *models.Database) error {
|
func (w *Writer) executeTableMode(db *models.Database) error {
|
||||||
|
var data []*TemplateData
|
||||||
for _, schema := range db.Schemas {
|
for _, schema := range db.Schemas {
|
||||||
for _, table := range schema.Tables {
|
for _, table := range schema.Tables {
|
||||||
data := NewTableData(table, schema, db, w.options.Metadata)
|
data = append(data, NewTableData(table, schema, db, w.options.Metadata))
|
||||||
output, err := w.executeTemplate(data)
|
}
|
||||||
|
}
|
||||||
|
return w.executeFiles(data)
|
||||||
|
}
|
||||||
|
|
||||||
|
type artifact struct{ name, content string }
|
||||||
|
|
||||||
|
func (w *Writer) outputRoot() string {
|
||||||
|
if w.options.OutputPath == "" {
|
||||||
|
return "."
|
||||||
|
}
|
||||||
|
return w.options.OutputPath
|
||||||
|
}
|
||||||
|
|
||||||
|
func (w *Writer) executeFiles(data []*TemplateData) error {
|
||||||
|
var files []artifact
|
||||||
|
seen := make(map[string]bool)
|
||||||
|
for _, item := range data {
|
||||||
|
name, err := w.renderFilename(item)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("failed to execute template for table %s.%s: %w", schema.Name, table.Name, err)
|
return err
|
||||||
}
|
}
|
||||||
|
key := strings.ToLower(filepath.ToSlash(name))
|
||||||
filename, err := w.generateFilename(data)
|
if seen[key] {
|
||||||
|
return fmt.Errorf("generated filename collision: %q", name)
|
||||||
|
}
|
||||||
|
seen[key] = true
|
||||||
|
content, err := w.executeTemplate(item)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("failed to generate filename for table %s.%s: %w", schema.Name, table.Name, err)
|
return err
|
||||||
}
|
}
|
||||||
|
files = append(files, artifact{name, content})
|
||||||
if err := w.writeOutput(output, filename); err != nil {
|
}
|
||||||
return fmt.Errorf("failed to write output for table %s.%s: %w", schema.Name, table.Name, err)
|
// Reject file/directory conflicts before creating any output.
|
||||||
|
for name := range seen {
|
||||||
|
for parent := filepath.ToSlash(filepath.Dir(name)); parent != "."; parent = filepath.ToSlash(filepath.Dir(parent)) {
|
||||||
|
if seen[parent] {
|
||||||
|
return fmt.Errorf("generated file/directory collision: %q", name)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
if len(files) == 0 {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
rootPath := w.outputRoot()
|
||||||
|
if err := safefile.CheckRoot(rootPath); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := os.MkdirAll(rootPath, 0o755); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
root, err := os.OpenRoot(rootPath)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer root.Close()
|
||||||
|
// Preflight every destination, including existing directory and symlink conflicts.
|
||||||
|
for _, file := range files {
|
||||||
|
if err := safefile.Check(root, file.name); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, file := range files {
|
||||||
|
if err := w.writeArtifact(root, file); err != nil {
|
||||||
|
return fmt.Errorf("write generated file %q: %w", file.name, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (w *Writer) writeArtifact(root *os.Root, file artifact) error {
|
||||||
|
if err := root.MkdirAll(filepath.Dir(file.name), 0o755); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
// Exclusive unpredictable staging avoids truncating a destination (or a link target).
|
||||||
|
var token [16]byte
|
||||||
|
if _, err := rand.Read(token[:]); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
temporary := ".relspec-" + hex.EncodeToString(token[:]) + ".tmp"
|
||||||
|
mode := os.FileMode(0o644)
|
||||||
|
if info, err := root.Lstat(file.name); err == nil && info.Mode().IsRegular() {
|
||||||
|
mode = info.Mode().Perm()
|
||||||
|
}
|
||||||
|
staged, err := root.OpenFile(temporary, os.O_WRONLY|os.O_CREATE|os.O_EXCL, mode)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer func() {
|
||||||
|
// Best-effort cleanup; a successful rename already removes the staging path.
|
||||||
|
_ = root.Remove(temporary)
|
||||||
|
}()
|
||||||
|
_, writeErr := staged.WriteString(file.content)
|
||||||
|
if err := errors.Join(writeErr, staged.Close()); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := safefile.Check(root, file.name); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return root.Rename(temporary, file.name)
|
||||||
|
}
|
||||||
|
|
||||||
// loadTemplate loads and parses the template file
|
// loadTemplate loads and parses the template file
|
||||||
func (w *Writer) loadTemplate() error {
|
func (w *Writer) loadTemplate() error {
|
||||||
@@ -245,43 +291,25 @@ func (w *Writer) executeTemplate(data *TemplateData) (string, error) {
|
|||||||
return buf.String(), nil
|
return buf.String(), nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// generateFilename generates a filename from the filename pattern
|
// renderFilename returns a validated path relative to the explicit output root.
|
||||||
func (w *Writer) generateFilename(data *TemplateData) (string, error) {
|
func (w *Writer) renderFilename(data *TemplateData) (string, error) {
|
||||||
// Parse filename pattern as a template
|
|
||||||
tmpl, err := template.New("filename").Funcs(w.funcMap).Parse(w.filenamePattern)
|
tmpl, err := template.New("filename").Funcs(w.funcMap).Parse(w.filenamePattern)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return "", fmt.Errorf("invalid filename pattern: %w", err)
|
return "", fmt.Errorf("invalid filename pattern: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Execute filename template
|
|
||||||
var buf bytes.Buffer
|
var buf bytes.Buffer
|
||||||
if err := tmpl.Execute(&buf, data); err != nil {
|
if err := tmpl.Execute(&buf, data); err != nil {
|
||||||
return "", fmt.Errorf("failed to generate filename: %w", err)
|
return "", fmt.Errorf("failed to generate filename: %w", err)
|
||||||
}
|
}
|
||||||
|
return safefile.Relative(buf.String())
|
||||||
filename := buf.String()
|
|
||||||
|
|
||||||
// If output path is a directory, join with generated filename
|
|
||||||
if w.options.OutputPath != "" {
|
|
||||||
// Check if output path is a directory
|
|
||||||
info, err := os.Stat(w.options.OutputPath)
|
|
||||||
if err == nil && info.IsDir() {
|
|
||||||
filename = filepath.Join(w.options.OutputPath, filename)
|
|
||||||
} else {
|
|
||||||
// If it doesn't exist, check if it looks like a directory (ends with /)
|
|
||||||
if strings.HasSuffix(w.options.OutputPath, string(filepath.Separator)) {
|
|
||||||
filename = filepath.Join(w.options.OutputPath, filename)
|
|
||||||
} else {
|
|
||||||
// Use output path as base directory
|
|
||||||
dir := filepath.Dir(w.options.OutputPath)
|
|
||||||
if dir != "." {
|
|
||||||
filename = filepath.Join(dir, filename)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
return filename, nil
|
func (w *Writer) generateFilename(data *TemplateData) (string, error) {
|
||||||
|
name, err := w.renderFilename(data)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
return filepath.Join(w.outputRoot(), name), nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// writeOutput writes the output to a file or stdout
|
// writeOutput writes the output to a file or stdout
|
||||||
|
|||||||
@@ -180,8 +180,8 @@ func TestWriterGenerateFilenameOutputPathForms(t *testing.T) {
|
|||||||
{"no output path", "", "users.txt"},
|
{"no output path", "", "users.txt"},
|
||||||
{"existing dir", dir, filepath.Join(dir, "users.txt")},
|
{"existing dir", dir, filepath.Join(dir, "users.txt")},
|
||||||
{"trailing separator", filepath.Join(dir, "new") + string(filepath.Separator), filepath.Join(dir, "new", "users.txt")},
|
{"trailing separator", filepath.Join(dir, "new") + string(filepath.Separator), filepath.Join(dir, "new", "users.txt")},
|
||||||
{"file path uses its dir", filepath.Join(dir, "x.out"), filepath.Join(dir, "users.txt")},
|
{"nonexistent path is root", filepath.Join(dir, "x.out"), filepath.Join(dir, "x.out", "users.txt")},
|
||||||
{"bare file name", "x.out", "users.txt"},
|
{"bare path is root", "x.out", filepath.Join("x.out", "users.txt")},
|
||||||
}
|
}
|
||||||
for _, tt := range tests {
|
for _, tt := range tests {
|
||||||
t.Run(tt.name, func(t *testing.T) {
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
@@ -217,3 +217,123 @@ func TestWriterWriteSchemaAndTable(t *testing.T) {
|
|||||||
t.Errorf("WriteTable: %q", data)
|
t.Errorf("WriteTable: %q", data)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestMultiFileRejectsUnsafeFilenamesBeforeWriting(t *testing.T) {
|
||||||
|
for _, name := range []string{"../escape", "/absolute", `..\escape`, `C:\escape`, `\\server\share\file`, "a/../escape", "a//escape", "CON", "nul.txt", "COM¹.txt", "file:stream", "trailing.", "trailing ", "line\nbreak"} {
|
||||||
|
t.Run(name, func(t *testing.T) {
|
||||||
|
out := filepath.Join(t.TempDir(), "new-output")
|
||||||
|
w, err := newTestWriter(t, "content", "schema", "{{.Name}}", out)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
db := models.InitDatabase("unsafe")
|
||||||
|
db.Schemas = []*models.Schema{models.InitSchema("valid"), models.InitSchema(name)}
|
||||||
|
if err := w.WriteDatabase(db); err == nil {
|
||||||
|
t.Fatal("expected rejected filename")
|
||||||
|
}
|
||||||
|
if _, err := os.Stat(out); !os.IsNotExist(err) {
|
||||||
|
t.Fatalf("validation failure must not create output: %v", err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMultiFileCollisionsDoNotOverwrite(t *testing.T) {
|
||||||
|
for _, names := range [][]string{{"same.txt", "same.txt"}, {"SAME.txt", "same.txt"}, {"file", "file/child.txt"}} {
|
||||||
|
out := t.TempDir()
|
||||||
|
seed := filepath.Join(out, "same.txt")
|
||||||
|
if err := os.WriteFile(seed, []byte("original"), 0o600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
w, err := newTestWriter(t, "replacement", "schema", "{{.Name}}", out)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
db := models.InitDatabase("collision")
|
||||||
|
for _, name := range names {
|
||||||
|
db.Schemas = append(db.Schemas, models.InitSchema(name))
|
||||||
|
}
|
||||||
|
if err := w.WriteDatabase(db); err == nil || !strings.Contains(err.Error(), "collision") {
|
||||||
|
t.Fatalf("expected collision error: %v", err)
|
||||||
|
}
|
||||||
|
got, err := os.ReadFile(seed)
|
||||||
|
if err != nil || string(got) != "original" {
|
||||||
|
t.Fatalf("existing output changed: %q %v", got, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMultiFileRejectsSymlinksBeforeAnyWrites(t *testing.T) {
|
||||||
|
for _, kind := range []string{"parent", "leaf", "root"} {
|
||||||
|
t.Run(kind, func(t *testing.T) {
|
||||||
|
out, external := t.TempDir(), t.TempDir()
|
||||||
|
secret := filepath.Join(external, "secret.txt")
|
||||||
|
if err := os.WriteFile(secret, []byte("original"), 0o600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
link, target := filepath.Join(out, "link"), external
|
||||||
|
pattern := "{{.Name}}/secret.txt"
|
||||||
|
if kind == "leaf" {
|
||||||
|
target = secret
|
||||||
|
pattern = "{{.Name}}"
|
||||||
|
}
|
||||||
|
if kind == "root" {
|
||||||
|
link = filepath.Join(out, "root")
|
||||||
|
out = link
|
||||||
|
pattern = "{{.Name}}"
|
||||||
|
}
|
||||||
|
if err := os.Symlink(target, link); err != nil {
|
||||||
|
t.Skip(err)
|
||||||
|
}
|
||||||
|
w, err := newTestWriter(t, "replacement", "schema", pattern, out)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
db := models.InitDatabase("links")
|
||||||
|
db.Schemas = []*models.Schema{models.InitSchema("first"), models.InitSchema("link")}
|
||||||
|
if err := w.WriteDatabase(db); err == nil {
|
||||||
|
t.Fatal("expected symlink rejection")
|
||||||
|
}
|
||||||
|
got, err := os.ReadFile(secret)
|
||||||
|
if err != nil || string(got) != "original" {
|
||||||
|
t.Fatalf("symlink target changed: %q %v", got, err)
|
||||||
|
}
|
||||||
|
if _, err := os.Stat(filepath.Join(out, "first")); !os.IsNotExist(err) {
|
||||||
|
t.Fatalf("preflight must precede writes: %v", err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMultiFileNewDirectoryAndNestedFiles(t *testing.T) {
|
||||||
|
out := filepath.Join(t.TempDir(), "new-output")
|
||||||
|
w, err := newTestWriter(t, "{{.Table.Name}}", "table", "{{.ParentSchema.Name}}/{{.Name}}.txt", out)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := w.WriteDatabase(modeDB()); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
got, err := os.ReadFile(filepath.Join(out, "a", "t1.txt"))
|
||||||
|
if err != nil || string(got) != "t1" {
|
||||||
|
t.Fatalf("nested output: %q %v", got, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMultiFileRootAncestorSymlinkRejected(t *testing.T) {
|
||||||
|
parent, external := t.TempDir(), t.TempDir()
|
||||||
|
link := filepath.Join(parent, "link")
|
||||||
|
if err := os.Symlink(external, link); err != nil {
|
||||||
|
t.Skip(err)
|
||||||
|
}
|
||||||
|
w, err := newTestWriter(t, "content", "schema", "{{.Name}}.txt", filepath.Join(link, "new"))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := w.WriteDatabase(modeDB()); err == nil {
|
||||||
|
t.Fatal("root ancestor symlink must fail")
|
||||||
|
}
|
||||||
|
if _, err := os.Stat(filepath.Join(external, "new")); !os.IsNotExist(err) {
|
||||||
|
t.Fatalf("must not create external directories: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user