Merge pull request 'fix(jobs): preserve writer and template output options' (#30) from issue-29-job-manifest-options into master

Reviewed-on: #30
Reviewed-by: Warky <2+warkanum@noreply@warky.dev>

.gitea/workflows/release.yml

@@ -63,6 +63,8 @@ jobs:
       - name: Build release binaries
         run: |
           VERSION="${{ github.event.inputs.tag || github.ref_name }}"
+          BUILD_DATE="$(date -u +"%Y-%m-%d %H:%M:%S UTC")"
+          LDFLAGS="-X 'git.warky.dev/wdevs/relspecgo/pkg/buildinfo.Version=${VERSION}' -X 'git.warky.dev/wdevs/relspecgo/pkg/buildinfo.BuildDate=${BUILD_DATE}'"
           for target in "linux/amd64" "linux/arm64" "darwin/amd64" "darwin/arm64" "windows/amd64"; do
             GOOS="${target%/*}"
             GOARCH="${target#*/}"
@@ -71,7 +73,7 @@ jobs:
             NAME="relspec-${GOOS}-${GOARCH}${EXT}"
             GOOS="$GOOS" GOARCH="$GOARCH" go build \
               -trimpath \
-              -ldflags "-X git.warky.dev/wdevs/relspecgo/cmd/relspec.version=${VERSION}" \
+              -ldflags "$LDFLAGS" \
               -o "$NAME" ./cmd/relspec
             echo "Built $NAME"
           done
@@ -234,11 +236,12 @@ jobs:
         run: |
           VERSION="${{ github.event.inputs.tag || github.ref_name }}"
           PKGVER="${VERSION#v}"
+          BUILD_DATE="$(date -u +"%Y-%m-%d %H:%M:%S UTC")"
 
           for GOARCH in amd64 arm64; do
             GOOS=linux GOARCH=$GOARCH go build \
               -trimpath \
-              -ldflags "-X git.warky.dev/wdevs/relspecgo/cmd/relspec.version=${PKGVER}" \
+              -ldflags "-X 'git.warky.dev/wdevs/relspecgo/pkg/buildinfo.Version=v${PKGVER}' -X 'git.warky.dev/wdevs/relspecgo/pkg/buildinfo.BuildDate=${BUILD_DATE}'" \
               -o relspec ./cmd/relspec
 
             PKGDIR="relspec_${PKGVER}_${GOARCH}"
@@ -309,7 +312,13 @@ jobs:
             rockylinux:9 \
             bash -lc "
               set -euo pipefail
-              dnf install -y rpm-build git &&
+              # Avoid transient/out-of-sync mirrors while bootstrapping the RPM builder.
+              sed -i \
+                -e 's|^mirrorlist=|#mirrorlist=|' \
+                -e 's|^#baseurl=http://dl.rockylinux.org|baseurl=https://dl.rockylinux.org|' \
+                /etc/yum.repos.d/rocky*.repo
+              dnf clean all
+              dnf -y --setopt=timeout=60 --setopt=retries=5 install rpm-build git
               curl -fsSL https://go.dev/dl/go\${GO_VER}.linux-amd64.tar.gz | tar -C /usr/local -xz &&
               export PATH=\$PATH:/usr/local/go/bin &&
               mkdir -p ~/rpmbuild/{BUILD,BUILDROOT,RPMS,SOURCES,SPECS,SRPMS} &&

Makefile

@@ -20,7 +20,9 @@ STATICCHECK   = go run honnef.co/go/tools/cmd/staticcheck@latest
 GOVULNCHECK   = go run golang.org/x/vuln/cmd/govulncheck@latest
 
 # Version information
-VERSION := $(shell git describe --tags --always --dirty 2>/dev/null || echo "dev")
+# VERSION file is the single source of truth (bumped by `make release-version`).
+# Falls back to git describe for ad-hoc builds where VERSION hasn't been committed yet.
+VERSION := $(shell [ -f VERSION ] && echo "v$$(cat VERSION | tr -d '[:space:]')" || git describe --tags --always --dirty 2>/dev/null || echo "dev")
 BUILD_DATE := $(shell date -u +"%Y-%m-%d %H:%M:%S UTC")
 LDFLAGS := -X 'git.warky.dev/wdevs/relspecgo/pkg/buildinfo.Version=$(VERSION)' -X 'git.warky.dev/wdevs/relspecgo/pkg/buildinfo.BuildDate=$(BUILD_DATE)'
 
@@ -223,6 +225,15 @@ release: lint fmt-check test build ## Run lint, format check, tests, build, then
 		echo "Creating new release: $$version"; \
 		commit_logs=$$(git log "$${latest_tag}..HEAD" --pretty=format:"- %s" --no-merges); \
 	fi; \
+	PKGVER=$${version#v}; \
+	echo "Updating package metadata to $$PKGVER..."; \
+	echo "$$PKGVER" > VERSION; \
+	sed -i "s/^pkgver=.*/pkgver=$$PKGVER/" linux/arch/PKGBUILD; \
+	sed -i "s/^Version:.*/Version:        $$PKGVER/" linux/centos/relspec.spec; \
+	git add VERSION linux/arch/PKGBUILD linux/centos/relspec.spec; \
+	if ! git diff --cached --quiet; then \
+		git commit -m "chore(release): update package version to $$PKGVER"; \
+	fi; \
 	if [ -z "$$commit_logs" ]; then \
 		tag_message="Release $$version"; \
 	else \
@@ -232,17 +243,18 @@ release: lint fmt-check test build ## Run lint, format check, tests, build, then
 	git push origin "$$version"; \
 	echo "Tag $$version created and pushed to remote repository."
 
-release-version: ## Auto-increment patch version, update package files, commit, tag, and push
+release-version: ## Auto-increment patch version, update VERSION/package files, commit, tag, and push
-	@CURRENT=$$(git describe --tags --abbrev=0 2>/dev/null || echo "v0.0.0"); \
+	@CURRENT=$$([ -f VERSION ] && cat VERSION | tr -d '[:space:]' || (git describe --tags --abbrev=0 2>/dev/null | sed 's/^v//') || echo "0.0.0"); \
-	MAJOR=$$(echo $$CURRENT | sed 's/v\([0-9]*\)\.\([0-9]*\)\.\([0-9]*\).*/\1/'); \
+	MAJOR=$$(echo $$CURRENT | cut -d. -f1); \
-	MINOR=$$(echo $$CURRENT | sed 's/v\([0-9]*\)\.\([0-9]*\)\.\([0-9]*\).*/\2/'); \
+	MINOR=$$(echo $$CURRENT | cut -d. -f2); \
-	PATCH=$$(echo $$CURRENT | sed 's/v\([0-9]*\)\.\([0-9]*\)\.\([0-9]*\).*/\3/'); \
+	PATCH=$$(echo $$CURRENT | cut -d. -f3); \
-	NEXT="v$$MAJOR.$$MINOR.$$((PATCH + 1))"; \
 	PKGVER="$$MAJOR.$$MINOR.$$((PATCH + 1))"; \
-	echo "Current: $$CURRENT → Next: $$NEXT"; \
+	NEXT="v$$PKGVER"; \
+	echo "Current: v$$CURRENT → Next: $$NEXT"; \
+	echo "$$PKGVER" > VERSION; \
 	sed -i "s/^pkgver=.*/pkgver=$$PKGVER/" linux/arch/PKGBUILD; \
 	sed -i "s/^Version:.*/Version:        $$PKGVER/" linux/centos/relspec.spec; \
-	git add linux/arch/PKGBUILD linux/centos/relspec.spec; \
+	git add VERSION linux/arch/PKGBUILD linux/centos/relspec.spec; \
 	git commit -m "chore(release): update package version to $$PKGVER"; \
 	git tag -a "$$NEXT" -m "Release $$NEXT"; \
 	git push origin HEAD "$$NEXT"; \

README.md

@@ -126,7 +126,7 @@ relspec job run build-schema
 version: 1
 jobs:
   build-schema:
-    command: convert            # closed allow-list: convert | merge | scripts-list
+    command: convert            # closed allow-list: convert | merge | split | scripts-list | scripts-exec | templ | inspect | diff
     description: Merge the DBML sources and emit PostgreSQL DDL
     inputs:
       - path: schema/core.dbml
@@ -147,7 +147,9 @@ resolved relative to the job file and may not escape it, and remote database
 credentials are referenced by environment-variable name (`conn_env:`) and
 redacted from logs. The whole plan — unknown commands/formats, duplicate job
 names, missing inputs, path traversal, dependency cycles — is validated before
-any job runs. See [docs/JOB_FILES.md](docs/JOB_FILES.md).
+any job runs. Path-like fields also support `${NAME}` environment-variable
+references, which are expanded and safety-checked during pre-flight. See
+[docs/JOB_FILES.md](docs/JOB_FILES.md).
 
 ### `edit` — Interactive TUI editor
 

VERSION

@@ -0,0 +1 @@
+1.0.82

cmd/relspec/job.go

@@ -60,8 +60,9 @@ inputs, output and options:
       logfile: .relspec/log/build-schema.log
 
 Rules and guarantees:
-  - command is a closed allow-list (convert, merge, scripts-list, templ). Arbitrary
+  - command is a closed allow-list (convert, merge, split, scripts-list,
-    shell strings are never executed.
+    scripts-exec, templ, inspect, diff). Arbitrary shell strings are never
+    executed.
   - Every path is relative to the directory holding the job file and may not
     escape it. Absolute and home-relative paths are rejected.
   - Remote database credentials are referenced by environment-variable name
@@ -246,22 +247,37 @@ type resolvedInput struct {
 func preflightJob(j *jobs.Job, resolvedByName map[string]*resolvedJob) (*resolvedJob, error) {
 	root := j.Dir()
 	rj := &resolvedJob{job: j, root: root, logPolicy: j.ResolvedLogPolicy()}
+	expandPath := func(label, value string) (string, error) {
+		expanded, err := jobs.ExpandEnv(value)
+		if err != nil {
+			return "", fmt.Errorf("%s: %w", label, err)
+		}
+		return expanded, nil
+	}
 
 	if j.Logfile != "" {
-		p, err := jobs.SafeJoin(root, j.Logfile)
+		logfile, err := expandPath("logfile", j.Logfile)
+		if err != nil {
+			return nil, err
+		}
+		p, err := jobs.SafeJoin(root, logfile)
 		if err != nil {
 			return nil, fmt.Errorf("logfile: %w", err)
 		}
 		rj.logPath = p
 	}
 	if j.Template != "" {
-		p, err := jobs.SafeJoin(root, j.Template)
+		templatePath, err := expandPath("template", j.Template)
+		if err != nil {
+			return nil, err
+		}
+		p, err := jobs.SafeJoin(root, templatePath)
 		if err != nil {
 			return nil, fmt.Errorf("template: %w", err)
 		}
 		info, err := os.Stat(p)
 		if err != nil || info.IsDir() {
-			return nil, fmt.Errorf("template %q: not found or is a directory", j.Template)
+			return nil, fmt.Errorf("template %q: not found or is a directory", templatePath)
 		}
 		rj.templatePath = p
 	}
@@ -291,16 +307,20 @@ func preflightJob(j *jobs.Job, resolvedByName map[string]*resolvedJob) (*resolve
 			ri.connEnv = in.ConnEnv
 			rj.secrets = append(rj.secrets, v)
 		} else {
-			p, err := jobs.SafeJoin(root, in.Path)
+			inputPath, err := expandPath(fmt.Sprintf("input[%d]", i), in.Path)
+			if err != nil {
+				return nil, err
+			}
+			p, err := jobs.SafeJoin(root, inputPath)
 			if err != nil {
 				return nil, fmt.Errorf("input[%d]: %w", i, err)
 			}
 			info, err := os.Stat(p)
 			if err != nil {
-				return nil, fmt.Errorf("input[%d]: %s: file not found", i, in.Path)
+				return nil, fmt.Errorf("input[%d]: %s: file not found", i, inputPath)
 			}
 			if info.IsDir() {
-				return nil, fmt.Errorf("input[%d]: %s: is a directory, not a file", i, in.Path)
+				return nil, fmt.Errorf("input[%d]: %s: is a directory, not a file", i, inputPath)
 			}
 			ri.path = p
 		}
@@ -308,13 +328,17 @@ func preflightJob(j *jobs.Job, resolvedByName map[string]*resolvedJob) (*resolve
 	}
 
 	for _, d := range j.ScriptDirs {
-		p, err := jobs.SafeJoin(root, d)
+		scriptDir, err := expandPath("script_dir", d)
+		if err != nil {
+			return nil, err
+		}
+		p, err := jobs.SafeJoin(root, scriptDir)
 		if err != nil {
 			return nil, fmt.Errorf("script_dir %q: %w", d, err)
 		}
 		info, err := os.Stat(p)
 		if err != nil {
-			return nil, fmt.Errorf("script_dir %q: not found", d)
+			return nil, fmt.Errorf("script_dir %q: not found", scriptDir)
 		}
 		if !info.IsDir() {
 			return nil, fmt.Errorf("script_dir %q: not a directory", d)
@@ -332,25 +356,33 @@ func preflightJob(j *jobs.Job, resolvedByName map[string]*resolvedJob) (*resolve
 			rj.outputConnEnv = j.Output.ConnEnv
 			rj.secrets = append(rj.secrets, v)
 		} else {
-			p, err := jobs.SafeJoin(root, j.Output.Path)
+			outputPath, err := expandPath("output", j.Output.Path)
+			if err != nil {
+				return nil, err
+			}
+			p, err := jobs.SafeJoin(root, outputPath)
 			if err != nil {
 				return nil, fmt.Errorf("output: %w", err)
 			}
 			if _, err := os.Stat(p); err == nil && !j.Output.Overwrite {
-				return nil, fmt.Errorf("output %s already exists (set output.overwrite: true to replace it)", j.Output.Path)
+				return nil, fmt.Errorf("output %s already exists (set output.overwrite: true to replace it)", outputPath)
 			}
 			rj.outputPath = p
 		}
 	}
 
 	if j.Rules != "" {
-		p, err := jobs.SafeJoin(root, j.Rules)
+		rulesPath, err := expandPath("rules", j.Rules)
+		if err != nil {
+			return nil, err
+		}
+		p, err := jobs.SafeJoin(root, rulesPath)
 		if err != nil {
 			return nil, fmt.Errorf("rules: %w", err)
 		}
 		info, err := os.Stat(p)
 		if err != nil || info.IsDir() {
-			return nil, fmt.Errorf("rules %q: not found or is a directory", j.Rules)
+			return nil, fmt.Errorf("rules %q: not found or is a directory", rulesPath)
 		}
 		rj.rulesPath = p
 	}
@@ -358,12 +390,16 @@ func preflightJob(j *jobs.Job, resolvedByName map[string]*resolvedJob) (*resolve
 	if j.Report != nil {
 		rj.reportFormat = strings.ToLower(j.Report.Format)
 		if j.Report.Path != "" {
-			p, err := jobs.SafeJoin(root, j.Report.Path)
+			reportPath, err := expandPath("report", j.Report.Path)
+			if err != nil {
+				return nil, err
+			}
+			p, err := jobs.SafeJoin(root, reportPath)
 			if err != nil {
 				return nil, fmt.Errorf("report: %w", err)
 			}
 			if _, err := os.Stat(p); err == nil && !j.Report.Overwrite {
-				return nil, fmt.Errorf("report %s already exists (set report.overwrite: true to replace it)", j.Report.Path)
+				return nil, fmt.Errorf("report %s already exists (set report.overwrite: true to replace it)", reportPath)
 			}
 			rj.reportPath = p
 		}
@@ -805,7 +841,7 @@ func writeJobOutput(rj *resolvedJob, db *models.Database, lg *jobLogger) error {
 			return fmt.Errorf("database output is only supported for pgsql (got %q)", rj.job.Output.Format)
 		}
 		lg.logf("writing output to database env:%s", rj.outputConnEnv)
-		writerOpts := newWriterOptions("", o.Package, o.FlattenSchema, "", "", o.ContinueOnError)
+		writerOpts := newWriterOptions("", o.Package, o.FlattenSchema, o.Types, o.ArrayNullable, o.ContinueOnError)
 		writerOpts.Metadata = map[string]interface{}{"connection_string": rj.outputConn}
 		return wpgsql.NewWriter(writerOpts).WriteDatabase(db)
 	}
@@ -816,7 +852,7 @@ func writeJobOutput(rj *resolvedJob, db *models.Database, lg *jobLogger) error {
 	lg.logf("writing output: %s (%s)", rj.outputPath, format)
 
 	write := func(target string) error {
-		return writeDatabase(db, format, target, o.Package, o.Schema, o.FlattenSchema, "", "", o.ContinueOnError, "")
+		return writeDatabase(db, format, target, o.Package, o.Schema, o.FlattenSchema, o.Types, o.ArrayNullable, o.ContinueOnError, "")
 	}
 	// Single-file formats are written to a temp file and renamed into place so
 	// a failure never leaves a partial or truncated output. Directory-emitting

cmd/relspec/job_test.go

@@ -666,6 +666,7 @@ jobs:
         format: dbml
     template: templates/schema.tmpl
     output:
+      format: text
       path: build/schema.txt
       overwrite: true
 `)
@@ -682,3 +683,61 @@ jobs:
 		t.Fatalf("templ output missing users table: %s", out)
 	}
 }
+
+func TestJobRun_BunSqlTypesOption(t *testing.T) {
+	dir := jobFixture(t, `version: 1
+jobs:
+  models:
+    command: convert
+    inputs:
+      - path: schema/core.dbml
+        format: dbml
+    output:
+      format: bun
+      path: build/models
+    options:
+      package: models
+      types: sqltypes
+`)
+	set := mustLoadSet(t, filepath.Join(dir, "relspec.yml"))
+	if err := executeJobPlan(set, "models", false, false, &bytes.Buffer{}); err != nil {
+		t.Fatalf("execute Bun job: %v", err)
+	}
+	generated, err := os.ReadFile(filepath.Join(dir, "build", "models"))
+	if err != nil {
+		t.Fatalf("read Bun output: %v", err)
+	}
+	if !strings.Contains(string(generated), "git.warky.dev/wdevs/relspecgo/pkg/sqltypes") {
+		t.Fatalf("Bun output did not preserve options.types: %s", generated)
+	}
+}
+
+func TestJobValidation_TemplTextFormat(t *testing.T) {
+	validDir := jobFixture(t, `version: 1
+jobs:
+  docs:
+    command: templ
+    inputs: [{path: schema/core.dbml, format: dbml}]
+    template: schema.tmpl
+    output: {format: text, path: build/docs.txt}
+`)
+	valid := mustLoadSet(t, filepath.Join(validDir, "relspec.yml"))
+	if err := valid.Validate(); err != nil {
+		t.Fatalf("text output should be valid for templ: %v", err)
+	}
+	invalidDir := jobFixture(t, `version: 1
+jobs:
+  docs:
+    command: templ
+    inputs: [{path: schema/core.dbml, format: dbml}]
+    template: schema.tmpl
+    output: {format: json, path: build/docs.txt}
+`)
+	invalid, err := jobs.Load([]string{filepath.Join(invalidDir, "relspec.yml")})
+	if err != nil {
+		t.Fatalf("load invalid manifest: %v", err)
+	}
+	if err := invalid.Validate(); err == nil || !strings.Contains(err.Error(), "output.format: text") {
+		t.Fatalf("expected templ format rejection, got %v", err)
+	}
+}

docs/JOB_FILES.md

@@ -55,6 +55,10 @@ already forbid duplicate keys within a single file.
   `script_dirs[]`, `template`, `logfile`) is **relative to the directory
   containing the job file that declared the job**, not the process working
   directory.
+* Those path fields support `${NAME}` environment-variable references. They
+  are expanded immediately before execution; a missing variable is an error.
+  The expanded value is still subject to all relative-path and symlink safety
+  checks.
 * Absolute paths, `~`-relative paths and any path that resolves outside the job
   file directory (`../`, `a/../../b`, …) are **rejected during validation** —
   before anything runs.
@@ -188,6 +192,8 @@ jobs:
       flatten_schema: false
       schema: public
       package: models                          # for gorm/bun output
+      types: sqltypes                           # Bun/GORM nullable types: baselib|stdlib|sqltypes
+      array_nullable: pointer_slice             # Bun: slice|pointer_slice
       continue_on_error: false                 # pgsql / scripts-exec output
       skip_relations: false                    # merge only
       skip_enums: false
@@ -201,8 +207,10 @@ jobs:
 
 For `templ`, `inputs` use the same file or `pgsql`/`conn_env` source forms as
 schema conversion. `output` is optional (empty means stdout); when present it
-contains only `path` and `overwrite`, because templates do not select a schema
+contains `path` and `overwrite`, plus the explicit `format: text` marker. The
-writer format.
+marker is templ-only: it describes arbitrary text produced by a Go template,
+not a database schema format. It is optional for compatibility with older
+manifests and has no effect on template execution.
 
 A `from_job` input takes no `path`, `format` or `conn_env`: it resolves to the
 named job's `output.path` and inherits its format, and implies a dependency on

examples/jobs/README.md

@@ -0,0 +1,80 @@
+# RelSpec job examples
+
+Run these commands from this directory:
+
+```bash
+cd examples/jobs
+
+# List every job from relspec.yml and the additional relspec.*.yml files.
+relspec job list
+
+# Validate a job and print its resolved plan without reading or writing data.
+relspec job run build-schema --plan
+
+# Build PostgreSQL DDL from the two DBML inputs.
+relspec job run build-schema
+
+# Run a job's dependency chain. This runs build-schema first, then build-json.
+relspec job run build-json
+
+# Inspect build-json's output. The from_job input adds the dependency automatically.
+relspec job run lint-schema
+
+# Extract only the posts table.
+relspec job run posts-only
+
+# List migration scripts in deterministic order without connecting to a database.
+relspec job run migration-order
+
+# Render one TypeScript file per table using the example template.
+relspec job run generate-typescript
+
+# Compare the two example schemas and write the summary to the logfile.
+relspec job run compare-schemas
+
+# Use an environment variable in an output path.
+export VST=build/typescript
+relspec job run generate-typescript --plan
+relspec job run generate-typescript
+```
+
+Database jobs use environment variables for connection strings. Set them in
+the shell before running the jobs; do not put connection strings in YAML:
+
+```bash
+export SOURCE_DB_URL='postgres://user:password@localhost/source_db'
+export TARGET_DB_URL='postgres://user:password@localhost/target_db'
+
+# Read the source database and write a local DBML snapshot.
+relspec job run snapshot-database
+
+# Preview migration execution without connecting to PostgreSQL.
+relspec job run apply-migrations --plan
+
+# Execute the migrations against TARGET_DB_URL.
+relspec job run apply-migrations
+```
+
+The `apply-migrations` job demonstrates execution from multiple directories:
+
+```yaml
+script_dirs:
+  - migrations/core
+  - migrations/tenant
+output:
+  format: pgsql
+  conn_env: TARGET_DB_URL
+```
+
+RelSpec combines the SQL files from both folders in deterministic order before
+executing them against the configured PostgreSQL database.
+
+To use a different directory or manifest, pass `--dir` or `--file`:
+
+```bash
+relspec job list --dir /path/to/project
+relspec job run build-schema --file /path/to/project/relspec.yml --plan
+```
+
+See [../../docs/JOB_FILES.md](../../docs/JOB_FILES.md) for the complete job
+file reference.

examples/jobs/relspec.database.yml

@@ -0,0 +1,29 @@
+# Database-oriented examples. Set the referenced environment variables before
+# running these jobs; connection strings never belong in a job file.
+version: 1
+
+jobs:
+  snapshot-database:
+    command: convert
+    description: Read a PostgreSQL database and write a DBML snapshot
+    inputs:
+      - format: pgsql
+        conn_env: SOURCE_DB_URL
+    output:
+      format: dbml
+      path: build/database-snapshot.dbml
+      overwrite: true
+
+  apply-migrations:
+    command: scripts-exec
+    description: Execute migrations from core and tenant folders against PostgreSQL
+    # All SQL files from these directories are discovered in deterministic order.
+    script_dirs:
+      - migrations/core
+      - migrations/tenant
+    output:
+      format: pgsql
+      conn_env: TARGET_DB_URL
+    options:
+      continue_on_error: false
+    logfile: .relspec/log/apply-migrations.log

examples/jobs/relspec.reports.yml

@@ -0,0 +1,32 @@
+# Reporting and templating examples. These jobs are discovered together with
+# relspec.yml when running `relspec job list` from this directory.
+version: 1
+
+jobs:
+  generate-typescript:
+    command: templ
+    description: Render one TypeScript file per table from the merged schema
+    inputs:
+      - path: schema/core.dbml
+        format: dbml
+      - path: schema/tenant.dbml
+        format: dbml
+    template: templates/schema.tmpl
+    mode: table
+    filename_pattern: "{{.Name}}.ts"
+    output:
+      format: text
+      path: "${VST}"
+      overwrite: true
+
+  compare-schemas:
+    command: diff
+    description: Compare the two example schemas and print a summary
+    inputs:
+      - path: schema/core.dbml
+        format: dbml
+      - path: schema/tenant.dbml
+        format: dbml
+    report:
+      format: summary
+    logfile: .relspec/log/compare-schemas.log

examples/jobs/templates/schema.tmpl

@@ -0,0 +1,6 @@
+// Code generated by RelSpec. DO NOT EDIT.
+export interface {{.Name}} {
+{{- range values .Table.Columns}}
+  {{.Name}}: {{.Type}};
+{{- end}}
+}

linux/arch/PKGBUILD

@@ -1,6 +1,6 @@
 # Maintainer: Hein (Warky Devs) <hein@warky.dev>
 pkgname=relspec
-pkgver=1.0.74
+pkgver=1.0.82
 pkgrel=1
 pkgdesc="RelSpec is a comprehensive database relations management tool that reads, transforms, and writes database table specifications across multiple formats and ORMs."
 arch=('x86_64' 'aarch64')
@@ -15,7 +15,7 @@ build() {
     export CGO_ENABLED=0
     go build \
         -trimpath \
-        -ldflags "-X git.warky.dev/wdevs/relspecgo/cmd/relspec.version=$pkgver" \
+        -ldflags "-X 'git.warky.dev/wdevs/relspecgo/pkg/buildinfo.Version=v$pkgver' -X 'git.warky.dev/wdevs/relspecgo/pkg/buildinfo.BuildDate=$(date -u +"%Y-%m-%d %H:%M:%S UTC")'" \
         -o "$pkgname" ./cmd/relspec
 }
 

linux/centos/relspec.spec

@@ -1,5 +1,5 @@
 Name:           relspec
-Version:        1.0.74
+Version:        1.0.82
 Release:        1%{?dist}
 Summary:        RelSpec is a comprehensive database relations management tool that reads, transforms, and writes database table specifications across multiple formats and ORMs.
 
@@ -25,7 +25,7 @@ DBML, GraphQL, and more.
 export CGO_ENABLED=0
 go build \
     -trimpath \
-    -ldflags "-X git.warky.dev/wdevs/relspecgo/cmd/relspec.version=%{version}" \
+    -ldflags "-X 'git.warky.dev/wdevs/relspecgo/pkg/buildinfo.Version=v%{version}' -X 'git.warky.dev/wdevs/relspecgo/pkg/buildinfo.BuildDate=$(date -u +"%%Y-%%m-%%d %%H:%%M:%%S UTC")'" \
     -o %{name} ./cmd/relspec
 
 %install

pkg/jobs/jobs.go

@@ -19,6 +19,7 @@ import (
 	"fmt"
 	"os"
 	"path/filepath"
+	"regexp"
 	"sort"
 	"strconv"
 	"strings"
@@ -276,9 +277,13 @@ func parseHumanSize(s string) (int64, error) {
 
 // Options carries the subset of command flags a job file may set.
 type Options struct {
-	FlattenSchema   bool   `yaml:"flatten_schema"`
+	FlattenSchema bool   `yaml:"flatten_schema"`
-	Schema          string `yaml:"schema"`
+	Schema        string `yaml:"schema"`
-	Package         string `yaml:"package"`
+	Package       string `yaml:"package"`
+	// Types and ArrayNullable mirror the existing convert/split writer flags.
+	// They are passed through only to writers that already support them.
+	Types           string `yaml:"types"`
+	ArrayNullable   string `yaml:"array_nullable"`
 	ContinueOnError bool   `yaml:"continue_on_error"`
 	SkipRelations   bool   `yaml:"skip_relations"`
 	SkipEnums       bool   `yaml:"skip_enums"`
@@ -568,7 +573,9 @@ func (j *Job) validate() []string {
 			e = append(e, "command \"templ\" does not support database output")
 		}
 		if j.Output != nil && j.Output.Format != "" {
-			e = append(e, "output.format is not valid for command \"templ\"")
+			if strings.ToLower(j.Output.Format) != "text" {
+				e = append(e, "command \"templ\" accepts only output.format: text")
+			}
 		}
 	case CommandSplit:
 		if len(j.Inputs) < 1 {
@@ -850,6 +857,29 @@ func SafeJoin(root, rel string) (string, error) {
 	return joined, nil
 }
 
+var envReferencePattern = regexp.MustCompile(`\$\{([A-Za-z_][A-Za-z0-9_]*)\}`)
+
+// ExpandEnv expands ${NAME} references using the current process environment.
+// It deliberately supports only shell-independent variable references; job
+// files are never passed through a shell. An unset variable is an error so a
+// typo cannot silently turn into a relative path.
+func ExpandEnv(value string) (string, error) {
+	var missing string
+	expanded := envReferencePattern.ReplaceAllStringFunc(value, func(reference string) string {
+		name := reference[2 : len(reference)-1]
+		resolved, ok := os.LookupEnv(name)
+		if !ok {
+			missing = name
+			return reference
+		}
+		return resolved
+	})
+	if missing != "" {
+		return "", fmt.Errorf("environment variable %q referenced by ${%s} is not set", missing, missing)
+	}
+	return expanded, nil
+}
+
 // deepestExistingAncestor returns p itself if it exists, otherwise the nearest
 // existing parent directory (falling back to the filesystem root).
 func deepestExistingAncestor(p string) string {

pkg/jobs/jobs_test.go

@@ -135,6 +135,21 @@ func TestParseHumanSize(t *testing.T) {
 	}
 }
 
+func TestExpandEnv(t *testing.T) {
+	t.Setenv("RELSPEC_TEST_ROOT", "build/output")
+	got, err := ExpandEnv("${RELSPEC_TEST_ROOT}/schema-${RELSPEC_TEST_ROOT}")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if want := "build/output/schema-build/output"; got != want {
+		t.Fatalf("ExpandEnv = %q, want %q", got, want)
+	}
+
+	if _, err := ExpandEnv("build/${RELSPEC_TEST_MISSING}"); err == nil {
+		t.Fatal("expected missing environment variable error")
+	}
+}
+
 func TestLoadRejectsDuplicateJobAcrossFiles(t *testing.T) {
 	dir := t.TempDir()
 	a := filepath.Join(dir, "relspec.yml")

pkg/models/models.go

@@ -227,23 +227,27 @@ func (d *Sequence) SQLName() string {
 
 // Column represents a table column
 type Column struct {
-	Name          string         `json:"name" yaml:"name" xml:"name"`
+	Name                 string         `json:"name" yaml:"name" xml:"name"`
-	Description   string         `json:"description,omitempty" yaml:"description,omitempty" xml:"description,omitempty"`
+	Description          string         `json:"description,omitempty" yaml:"description,omitempty" xml:"description,omitempty"`
-	Table         string         `json:"table" yaml:"table" xml:"table"`
+	Table                string         `json:"table" yaml:"table" xml:"table"`
-	Schema        string         `json:"schema" yaml:"schema" xml:"schema"`
+	Schema               string         `json:"schema" yaml:"schema" xml:"schema"`
-	Type          string         `json:"type" yaml:"type" xml:"type"`
+	Type                 string         `json:"type" yaml:"type" xml:"type"`
-	Length        int            `json:"length,omitempty" yaml:"length,omitempty" xml:"length,omitempty"`
+	Length               int            `json:"length,omitempty" yaml:"length,omitempty" xml:"length,omitempty"`
-	Precision     int            `json:"precision,omitempty" yaml:"precision,omitempty" xml:"precision,omitempty"`
+	Precision            int            `json:"precision,omitempty" yaml:"precision,omitempty" xml:"precision,omitempty"`
-	Scale         int            `json:"scale,omitempty" yaml:"scale,omitempty" xml:"scale,omitempty"`
+	Scale                int            `json:"scale,omitempty" yaml:"scale,omitempty" xml:"scale,omitempty"`
-	NotNull       bool           `json:"not_null" yaml:"not_null" xml:"not_null"`
+	NotNull              bool           `json:"not_null" yaml:"not_null" xml:"not_null"`
-	Default       any            `json:"default,omitempty" yaml:"default,omitempty" xml:"default,omitempty"`
+	Default              any            `json:"default,omitempty" yaml:"default,omitempty" xml:"default,omitempty"`
-	AutoIncrement bool           `json:"auto_increment" yaml:"auto_increment" xml:"auto_increment"`
+	AutoIncrement        bool           `json:"auto_increment" yaml:"auto_increment" xml:"auto_increment"`
-	IsPrimaryKey  bool           `json:"is_primary_key" yaml:"is_primary_key" xml:"is_primary_key"`
+	IsPrimaryKey         bool           `json:"is_primary_key" yaml:"is_primary_key" xml:"is_primary_key"`
-	Comment       string         `json:"comment,omitempty" yaml:"comment,omitempty" xml:"comment,omitempty"`
+	Comment              string         `json:"comment,omitempty" yaml:"comment,omitempty" xml:"comment,omitempty"`
-	Collation     string         `json:"collation,omitempty" yaml:"collation,omitempty" xml:"collation,omitempty"`
+	Collation            string         `json:"collation,omitempty" yaml:"collation,omitempty" xml:"collation,omitempty"`
-	Metadata      map[string]any `json:"metadata,omitempty" yaml:"metadata,omitempty" xml:"-"`
+	Metadata             map[string]any `json:"metadata,omitempty" yaml:"metadata,omitempty" xml:"-"`
-	Sequence      uint           `json:"sequence,omitempty" yaml:"sequence,omitempty" xml:"sequence,omitempty"`
+	Sequence             uint           `json:"sequence,omitempty" yaml:"sequence,omitempty" xml:"sequence,omitempty"`
-	GUID          string         `json:"guid" yaml:"guid" xml:"guid"`
+	GUID                 string         `json:"guid" yaml:"guid" xml:"guid"`
+	Generated            bool           `json:"generated,omitempty" yaml:"generated,omitempty" xml:"generated,omitempty"`
+	GenerationExpression string         `json:"generation_expression,omitempty" yaml:"generation_expression,omitempty" xml:"generation_expression,omitempty"`
+	Identity             bool           `json:"identity,omitempty" yaml:"identity,omitempty" xml:"identity,omitempty"`
+	IdentityGeneration   string         `json:"identity_generation,omitempty" yaml:"identity_generation,omitempty" xml:"identity_generation,omitempty"` // "ALWAYS" or "BY DEFAULT"
 }
 
 // SQLName returns the column name in lowercase for SQL compatibility.

pkg/readers/pgsql/queries.go

@@ -243,7 +243,11 @@ func (r *Reader) queryColumns(schemaName string) (map[string]map[string]*models.
 			c.numeric_scale,
 			c.udt_name,
 			pg_catalog.format_type(a.atttypid, a.atttypmod) as formatted_data_type,
-			col_description((c.table_schema||'.'||c.table_name)::regclass, c.ordinal_position) as description
+			col_description((c.table_schema||'.'||c.table_name)::regclass, c.ordinal_position) as description,
+			c.is_generated,
+			c.generation_expression,
+			c.is_identity,
+			c.identity_generation
 		FROM information_schema.columns c
 		JOIN pg_catalog.pg_namespace n
 			ON n.nspname = c.table_schema
@@ -268,17 +272,36 @@ func (r *Reader) queryColumns(schemaName string) (map[string]map[string]*models.
 	columnsMap := make(map[string]map[string]*models.Column)
 
 	for rows.Next() {
-		var schema, tableName, columnName, isNullable, dataType, udtName, formattedDataType string
+		var schema, tableName, columnName, isNullable, dataType, udtName, formattedDataType, isGenerated, isIdentity string
 		var ordinalPosition int
-		var columnDefault, description *string
+		var columnDefault, description, generationExpression, identityGeneration *string
 		var charMaxLength, numPrecision, numScale *int
 
-		if err := rows.Scan(&schema, &tableName, &columnName, &ordinalPosition, &columnDefault, &isNullable, &dataType, &charMaxLength, &numPrecision, &numScale, &udtName, &formattedDataType, &description); err != nil {
+		if err := rows.Scan(&schema, &tableName, &columnName, &ordinalPosition, &columnDefault, &isNullable, &dataType, &charMaxLength, &numPrecision, &numScale, &udtName, &formattedDataType, &description, &isGenerated, &generationExpression, &isIdentity, &identityGeneration); err != nil {
 			return nil, err
 		}
 
 		column := models.InitColumn(columnName, tableName, schema)
 
+		// GENERATED ALWAYS ... STORED columns are computed by Postgres and cannot
+		// have their default altered/dropped like a regular column.
+		column.Generated = isGenerated == "ALWAYS"
+		if generationExpression != nil {
+			column.GenerationExpression = *generationExpression
+		}
+
+		// GENERATED { ALWAYS | BY DEFAULT } AS IDENTITY columns are driven by an
+		// internal sequence rather than a literal default (unlike serial columns,
+		// they carry no pg_attrdef row at all), so they need the same DB-side
+		// handling as generated columns even though the underlying mechanism differs.
+		column.Identity = isIdentity == "YES"
+		if identityGeneration != nil {
+			column.IdentityGeneration = strings.ToUpper(strings.TrimSpace(*identityGeneration))
+		}
+		if column.Identity {
+			column.AutoIncrement = true
+		}
+
 		// Check if this is a serial type (has nextval default)
 		hasNextval := false
 		if columnDefault != nil {
@@ -301,6 +324,24 @@ func (r *Reader) queryColumns(schemaName string) (map[string]map[string]*models.
 			column.Description = *description
 		}
 
+		if column.Generated {
+			note := "GENERATED ALWAYS AS (" + column.GenerationExpression + ") STORED"
+			if column.Description != "" {
+				column.Description = column.Description + " " + note
+			} else {
+				column.Description = note
+			}
+		}
+
+		if column.Identity {
+			note := "GENERATED " + column.IdentityGeneration + " AS IDENTITY"
+			if column.Description != "" {
+				column.Description = column.Description + " " + note
+			} else {
+				column.Description = note
+			}
+		}
+
 		if charMaxLength != nil {
 			column.Length = *charMaxLength
 		}

pkg/writers/pgsql/migration_writer.go

@@ -433,6 +433,16 @@ func (w *MigrationWriter) generateAlterTableScripts(schema *models.Schema, model
 				Body:       sql,
 			}
 			scripts = append(scripts, script)
+		} else if currentCol.Generated || currentCol.Identity {
+			// GENERATED ALWAYS ... STORED columns and GENERATED ... AS IDENTITY
+			// columns are both driven by Postgres-side machinery (an expression, or
+			// a backing sequence + attidentity flag) that DBML cannot express, so
+			// the model side always looks like drift (missing type modifiers, no
+			// default, NOT NULL mismatches, etc). None of their properties are
+			// alterable the normal way (e.g. dropping a generated column's default
+			// requires DROP EXPRESSION, not DROP DEFAULT; identity columns have no
+			// default to alter at all), so these DB-side-only columns are left
+			// untouched rather than diffed.
 		} else if !columnsEqual(modelCol, currentCol) {
 			// Column exists but properties changed
 			if !columnTypesEqual(modelCol, currentCol) {

pkg/writers/pgsql/migration_writer_test.go

@@ -219,6 +219,91 @@ func TestWriteMigration_AltersColumnNullabilityWhenNotNullDiffers(t *testing.T)
 	}
 }
 
+func TestWriteMigration_DoesNotDropDefaultOnGeneratedColumn(t *testing.T) {
+	current := models.InitDatabase("testdb")
+	currentSchema := models.InitSchema("public")
+	currentTable := models.InitTable("info_city", "public")
+	currentGeom := models.InitColumn("geom", "info_city", "public")
+	currentGeom.Type = "text"
+	currentGeom.Generated = true
+	currentGeom.GenerationExpression = "st_makepoint(lon, lat)"
+	currentTable.Columns["geom"] = currentGeom
+	currentSchema.Tables = append(currentSchema.Tables, currentTable)
+	current.Schemas = append(current.Schemas, currentSchema)
+
+	model := models.InitDatabase("testdb")
+	modelSchema := models.InitSchema("public")
+	modelTable := models.InitTable("info_city", "public")
+	modelGeom := models.InitColumn("geom", "info_city", "public")
+	modelGeom.Type = "text"
+	// DBML cannot express GENERATED ALWAYS columns, so the model declares
+	// this as a plain column with no default — it must not be treated as a
+	// default-value drift against the DB-side generated column.
+	modelTable.Columns["geom"] = modelGeom
+	modelSchema.Tables = append(modelSchema.Tables, modelTable)
+	model.Schemas = append(model.Schemas, modelSchema)
+
+	var buf bytes.Buffer
+	writer, err := NewMigrationWriter(&writers.WriterOptions{})
+	if err != nil {
+		t.Fatalf("Failed to create writer: %v", err)
+	}
+	writer.writer = &buf
+
+	if err := writer.WriteMigration(model, current); err != nil {
+		t.Fatalf("WriteMigration failed: %v", err)
+	}
+
+	output := buf.String()
+	if strings.Contains(output, "DROP DEFAULT") {
+		t.Fatalf("did not expect DROP DEFAULT against a generated column, got:\n%s", output)
+	}
+}
+
+func TestWriteMigration_DoesNotAlterIdentityColumn(t *testing.T) {
+	current := models.InitDatabase("testdb")
+	currentSchema := models.InitSchema("public")
+	currentTable := models.InitTable("users", "public")
+	currentID := models.InitColumn("id", "users", "public")
+	currentID.Type = "bigint"
+	currentID.NotNull = true
+	currentID.Identity = true
+	currentID.IdentityGeneration = "ALWAYS"
+	currentTable.Columns["id"] = currentID
+	currentSchema.Tables = append(currentSchema.Tables, currentTable)
+	current.Schemas = append(current.Schemas, currentSchema)
+
+	model := models.InitDatabase("testdb")
+	modelSchema := models.InitSchema("public")
+	modelTable := models.InitTable("users", "public")
+	modelID := models.InitColumn("id", "users", "public")
+	modelID.Type = "bigint"
+	// DBML cannot express GENERATED ... AS IDENTITY, so the model declares this
+	// as a plain nullable column — it must not be diffed as NOT NULL drift
+	// against the DB-side identity column (Postgres rejects DROP NOT NULL on
+	// an identity column).
+	modelID.NotNull = false
+	modelTable.Columns["id"] = modelID
+	modelSchema.Tables = append(modelSchema.Tables, modelTable)
+	model.Schemas = append(model.Schemas, modelSchema)
+
+	var buf bytes.Buffer
+	writer, err := NewMigrationWriter(&writers.WriterOptions{})
+	if err != nil {
+		t.Fatalf("Failed to create writer: %v", err)
+	}
+	writer.writer = &buf
+
+	if err := writer.WriteMigration(model, current); err != nil {
+		t.Fatalf("WriteMigration failed: %v", err)
+	}
+
+	output := buf.String()
+	if strings.Contains(output, "DROP NOT NULL") {
+		t.Fatalf("did not expect DROP NOT NULL against an identity column, got:\n%s", output)
+	}
+}
+
 func TestWriteMigration_UsesStorageTypeForSerialAlterStatements(t *testing.T) {
 	current := models.InitDatabase("testdb")
 	currentSchema := models.InitSchema("public")

pkg/writers/pgsql/writer.go

@@ -462,6 +462,12 @@ func (w *Writer) GenerateAlterColumnTypeStatements(schema *models.Schema) ([]str
 	for _, table := range schema.Tables {
 		columns := getSortedColumns(table.Columns)
 		for _, col := range columns {
+			if col.Generated || col.Identity {
+				// Type is derived from the generation expression (generated columns) or
+				// tied to the backing sequence (identity columns); leave both alone here
+				// rather than risk an ALTER COLUMN TYPE Postgres won't accept cleanly.
+				continue
+			}
 			targetType := effectiveAlterColumnSQLType(col)
 			stmt, err := w.executor.ExecuteAlterColumnTypeWithCheck(AlterColumnTypeWithCheckData{
 				SchemaName:      schema.Name,
@@ -492,6 +498,14 @@ func (w *Writer) GenerateAlterColumnDefaultStatements(schema *models.Schema) ([]
 	for _, table := range schema.Tables {
 		columns := getSortedColumns(table.Columns)
 		for _, col := range columns {
+			if col.Generated || col.Identity {
+				// Generated columns: Postgres stores the generation expression in
+				// pg_attrdef just like a real default, so the guard template would see
+				// it as a stray default and emit DROP DEFAULT, which Postgres rejects.
+				// Identity columns never carry a DEFAULT at all (they're driven by a
+				// backing sequence), so there is nothing for this generator to manage.
+				continue
+			}
 			setDefault, defaultVal := formatColumnDefaultSQL(col)
 			stmt, err := w.executor.ExecuteAlterColumnDefaultWithCheck(AlterColumnDefaultWithCheckData{
 				SchemaName:   schema.Name,
@@ -605,13 +619,25 @@ func (w *Writer) generateColumnDefinition(col *models.Column) string {
 
 	parts = append(parts, effectiveColumnSQLType(col))
 
+	// GENERATED ALWAYS ... STORED columns are computed from an expression and
+	// cannot also carry a DEFAULT, so this takes the place of the DEFAULT clause.
+	if col.Generated && col.GenerationExpression != "" {
+		parts = append(parts, fmt.Sprintf("GENERATED ALWAYS AS (%s) STORED", col.GenerationExpression))
+	}
+
+	// GENERATED { ALWAYS | BY DEFAULT } AS IDENTITY columns are driven by an
+	// internal sequence, not a literal DEFAULT, and cannot carry one either.
+	if col.Identity && col.IdentityGeneration != "" {
+		parts = append(parts, fmt.Sprintf("GENERATED %s AS IDENTITY", col.IdentityGeneration))
+	}
+
 	// NOT NULL
 	if col.NotNull {
 		parts = append(parts, "NOT NULL")
 	}
 
 	// DEFAULT
-	if col.Default != nil {
+	if !col.Generated && !col.Identity && col.Default != nil {
 		switch v := col.Default.(type) {
 		case string:
 			parts = append(parts, fmt.Sprintf("DEFAULT %s", writers.QuoteDefaultValue(stripBackticks(v), col.Type)))

pkg/writers/pgsql/writer_test.go

@@ -1510,3 +1510,42 @@ func TestIndexStorageParameters(t *testing.T) {
 		})
 	}
 }
+
+func TestGenerateColumnDefinition_GeneratedColumnEmitsGeneratedClauseNotDefault(t *testing.T) {
+	w := &Writer{}
+
+	col := models.InitColumn("geom", "info_city", "public")
+	col.Type = "text"
+	col.NotNull = true
+	col.Generated = true
+	col.GenerationExpression = "st_makepoint(lon, lat)"
+	// A generated column's introspected "default" is actually its generation
+	// expression surfaced via pg_attrdef; it must never be rendered as DEFAULT.
+	col.Default = "st_makepoint(lon, lat)"
+
+	got := w.generateColumnDefinition(col)
+
+	want := "geom text GENERATED ALWAYS AS (st_makepoint(lon, lat)) STORED NOT NULL"
+	if got != want {
+		t.Fatalf("generateColumnDefinition() = %q, want %q", got, want)
+	}
+}
+
+func TestGenerateColumnDefinition_IdentityColumnEmitsIdentityClauseNotDefault(t *testing.T) {
+	w := &Writer{}
+
+	col := models.InitColumn("id", "users", "public")
+	col.Type = "bigint"
+	col.NotNull = true
+	col.Identity = true
+	col.IdentityGeneration = "ALWAYS"
+	// Identity columns never carry a literal default; this must never surface.
+	col.Default = "some-stray-value"
+
+	got := w.generateColumnDefinition(col)
+
+	want := "id bigint GENERATED ALWAYS AS IDENTITY NOT NULL"
+	if got != want {
+		t.Fatalf("generateColumnDefinition() = %q, want %q", got, want)
+	}
+}