From ea70e19a467be7a1f98bb078b05e87e0e283ff23 Mon Sep 17 00:00:00 2001 From: Hein Date: Sun, 20 Sep 2026 20:08:20 +0200 Subject: [PATCH] feat(jobs): expand environment variables in paths --- README.md | 6 ++- cmd/relspec/job.go | 68 ++++++++++++++++++------ docs/JOB_FILES.md | 4 ++ examples/jobs/README.md | 80 +++++++++++++++++++++++++++++ examples/jobs/relspec.database.yml | 29 +++++++++++ examples/jobs/relspec.reports.yml | 31 +++++++++++ examples/jobs/templates/schema.tmpl | 6 +++ pkg/jobs/jobs.go | 24 +++++++++ pkg/jobs/jobs_test.go | 15 ++++++ 9 files changed, 245 insertions(+), 18 deletions(-) create mode 100644 examples/jobs/README.md create mode 100644 examples/jobs/relspec.database.yml create mode 100644 examples/jobs/relspec.reports.yml create mode 100644 examples/jobs/templates/schema.tmpl diff --git a/README.md b/README.md index 5c9a416..69f4b44 100644 --- a/README.md +++ b/README.md @@ -126,7 +126,7 @@ relspec job run build-schema version: 1 jobs: build-schema: - command: convert # closed allow-list: convert | merge | scripts-list + command: convert # closed allow-list: convert | merge | split | scripts-list | scripts-exec | templ | inspect | diff description: Merge the DBML sources and emit PostgreSQL DDL inputs: - path: schema/core.dbml @@ -147,7 +147,9 @@ resolved relative to the job file and may not escape it, and remote database credentials are referenced by environment-variable name (`conn_env:`) and redacted from logs. The whole plan — unknown commands/formats, duplicate job names, missing inputs, path traversal, dependency cycles — is validated before -any job runs. See [docs/JOB_FILES.md](docs/JOB_FILES.md). +any job runs. Path-like fields also support `${NAME}` environment-variable +references, which are expanded and safety-checked during pre-flight. See +[docs/JOB_FILES.md](docs/JOB_FILES.md). ### `edit` — Interactive TUI editor diff --git a/cmd/relspec/job.go b/cmd/relspec/job.go index dd3d9d0..816d6a2 100644 --- a/cmd/relspec/job.go +++ b/cmd/relspec/job.go @@ -60,8 +60,9 @@ inputs, output and options: logfile: .relspec/log/build-schema.log Rules and guarantees: - - command is a closed allow-list (convert, merge, scripts-list, templ). Arbitrary - shell strings are never executed. + - command is a closed allow-list (convert, merge, split, scripts-list, + scripts-exec, templ, inspect, diff). Arbitrary shell strings are never + executed. - Every path is relative to the directory holding the job file and may not escape it. Absolute and home-relative paths are rejected. - Remote database credentials are referenced by environment-variable name @@ -246,22 +247,37 @@ type resolvedInput struct { func preflightJob(j *jobs.Job, resolvedByName map[string]*resolvedJob) (*resolvedJob, error) { root := j.Dir() rj := &resolvedJob{job: j, root: root, logPolicy: j.ResolvedLogPolicy()} + expandPath := func(label, value string) (string, error) { + expanded, err := jobs.ExpandEnv(value) + if err != nil { + return "", fmt.Errorf("%s: %w", label, err) + } + return expanded, nil + } if j.Logfile != "" { - p, err := jobs.SafeJoin(root, j.Logfile) + logfile, err := expandPath("logfile", j.Logfile) + if err != nil { + return nil, err + } + p, err := jobs.SafeJoin(root, logfile) if err != nil { return nil, fmt.Errorf("logfile: %w", err) } rj.logPath = p } if j.Template != "" { - p, err := jobs.SafeJoin(root, j.Template) + templatePath, err := expandPath("template", j.Template) + if err != nil { + return nil, err + } + p, err := jobs.SafeJoin(root, templatePath) if err != nil { return nil, fmt.Errorf("template: %w", err) } info, err := os.Stat(p) if err != nil || info.IsDir() { - return nil, fmt.Errorf("template %q: not found or is a directory", j.Template) + return nil, fmt.Errorf("template %q: not found or is a directory", templatePath) } rj.templatePath = p } @@ -291,16 +307,20 @@ func preflightJob(j *jobs.Job, resolvedByName map[string]*resolvedJob) (*resolve ri.connEnv = in.ConnEnv rj.secrets = append(rj.secrets, v) } else { - p, err := jobs.SafeJoin(root, in.Path) + inputPath, err := expandPath(fmt.Sprintf("input[%d]", i), in.Path) + if err != nil { + return nil, err + } + p, err := jobs.SafeJoin(root, inputPath) if err != nil { return nil, fmt.Errorf("input[%d]: %w", i, err) } info, err := os.Stat(p) if err != nil { - return nil, fmt.Errorf("input[%d]: %s: file not found", i, in.Path) + return nil, fmt.Errorf("input[%d]: %s: file not found", i, inputPath) } if info.IsDir() { - return nil, fmt.Errorf("input[%d]: %s: is a directory, not a file", i, in.Path) + return nil, fmt.Errorf("input[%d]: %s: is a directory, not a file", i, inputPath) } ri.path = p } @@ -308,13 +328,17 @@ func preflightJob(j *jobs.Job, resolvedByName map[string]*resolvedJob) (*resolve } for _, d := range j.ScriptDirs { - p, err := jobs.SafeJoin(root, d) + scriptDir, err := expandPath("script_dir", d) + if err != nil { + return nil, err + } + p, err := jobs.SafeJoin(root, scriptDir) if err != nil { return nil, fmt.Errorf("script_dir %q: %w", d, err) } info, err := os.Stat(p) if err != nil { - return nil, fmt.Errorf("script_dir %q: not found", d) + return nil, fmt.Errorf("script_dir %q: not found", scriptDir) } if !info.IsDir() { return nil, fmt.Errorf("script_dir %q: not a directory", d) @@ -332,25 +356,33 @@ func preflightJob(j *jobs.Job, resolvedByName map[string]*resolvedJob) (*resolve rj.outputConnEnv = j.Output.ConnEnv rj.secrets = append(rj.secrets, v) } else { - p, err := jobs.SafeJoin(root, j.Output.Path) + outputPath, err := expandPath("output", j.Output.Path) + if err != nil { + return nil, err + } + p, err := jobs.SafeJoin(root, outputPath) if err != nil { return nil, fmt.Errorf("output: %w", err) } if _, err := os.Stat(p); err == nil && !j.Output.Overwrite { - return nil, fmt.Errorf("output %s already exists (set output.overwrite: true to replace it)", j.Output.Path) + return nil, fmt.Errorf("output %s already exists (set output.overwrite: true to replace it)", outputPath) } rj.outputPath = p } } if j.Rules != "" { - p, err := jobs.SafeJoin(root, j.Rules) + rulesPath, err := expandPath("rules", j.Rules) + if err != nil { + return nil, err + } + p, err := jobs.SafeJoin(root, rulesPath) if err != nil { return nil, fmt.Errorf("rules: %w", err) } info, err := os.Stat(p) if err != nil || info.IsDir() { - return nil, fmt.Errorf("rules %q: not found or is a directory", j.Rules) + return nil, fmt.Errorf("rules %q: not found or is a directory", rulesPath) } rj.rulesPath = p } @@ -358,12 +390,16 @@ func preflightJob(j *jobs.Job, resolvedByName map[string]*resolvedJob) (*resolve if j.Report != nil { rj.reportFormat = strings.ToLower(j.Report.Format) if j.Report.Path != "" { - p, err := jobs.SafeJoin(root, j.Report.Path) + reportPath, err := expandPath("report", j.Report.Path) + if err != nil { + return nil, err + } + p, err := jobs.SafeJoin(root, reportPath) if err != nil { return nil, fmt.Errorf("report: %w", err) } if _, err := os.Stat(p); err == nil && !j.Report.Overwrite { - return nil, fmt.Errorf("report %s already exists (set report.overwrite: true to replace it)", j.Report.Path) + return nil, fmt.Errorf("report %s already exists (set report.overwrite: true to replace it)", reportPath) } rj.reportPath = p } diff --git a/docs/JOB_FILES.md b/docs/JOB_FILES.md index f5713ba..4f911be 100644 --- a/docs/JOB_FILES.md +++ b/docs/JOB_FILES.md @@ -55,6 +55,10 @@ already forbid duplicate keys within a single file. `script_dirs[]`, `template`, `logfile`) is **relative to the directory containing the job file that declared the job**, not the process working directory. +* Those path fields support `${NAME}` environment-variable references. They + are expanded immediately before execution; a missing variable is an error. + The expanded value is still subject to all relative-path and symlink safety + checks. * Absolute paths, `~`-relative paths and any path that resolves outside the job file directory (`../`, `a/../../b`, …) are **rejected during validation** — before anything runs. diff --git a/examples/jobs/README.md b/examples/jobs/README.md new file mode 100644 index 0000000..f7c131c --- /dev/null +++ b/examples/jobs/README.md @@ -0,0 +1,80 @@ +# RelSpec job examples + +Run these commands from this directory: + +```bash +cd examples/jobs + +# List every job from relspec.yml and the additional relspec.*.yml files. +relspec job list + +# Validate a job and print its resolved plan without reading or writing data. +relspec job run build-schema --plan + +# Build PostgreSQL DDL from the two DBML inputs. +relspec job run build-schema + +# Run a job's dependency chain. This runs build-schema first, then build-json. +relspec job run build-json + +# Inspect build-json's output. The from_job input adds the dependency automatically. +relspec job run lint-schema + +# Extract only the posts table. +relspec job run posts-only + +# List migration scripts in deterministic order without connecting to a database. +relspec job run migration-order + +# Render one TypeScript file per table using the example template. +relspec job run generate-typescript + +# Compare the two example schemas and write the summary to the logfile. +relspec job run compare-schemas + +# Use an environment variable in an output path. +export VST=build/typescript +relspec job run generate-typescript --plan +relspec job run generate-typescript +``` + +Database jobs use environment variables for connection strings. Set them in +the shell before running the jobs; do not put connection strings in YAML: + +```bash +export SOURCE_DB_URL='postgres://user:password@localhost/source_db' +export TARGET_DB_URL='postgres://user:password@localhost/target_db' + +# Read the source database and write a local DBML snapshot. +relspec job run snapshot-database + +# Preview migration execution without connecting to PostgreSQL. +relspec job run apply-migrations --plan + +# Execute the migrations against TARGET_DB_URL. +relspec job run apply-migrations +``` + +The `apply-migrations` job demonstrates execution from multiple directories: + +```yaml +script_dirs: + - migrations/core + - migrations/tenant +output: + format: pgsql + conn_env: TARGET_DB_URL +``` + +RelSpec combines the SQL files from both folders in deterministic order before +executing them against the configured PostgreSQL database. + +To use a different directory or manifest, pass `--dir` or `--file`: + +```bash +relspec job list --dir /path/to/project +relspec job run build-schema --file /path/to/project/relspec.yml --plan +``` + +See [../../docs/JOB_FILES.md](../../docs/JOB_FILES.md) for the complete job +file reference. diff --git a/examples/jobs/relspec.database.yml b/examples/jobs/relspec.database.yml new file mode 100644 index 0000000..1a28b5e --- /dev/null +++ b/examples/jobs/relspec.database.yml @@ -0,0 +1,29 @@ +# Database-oriented examples. Set the referenced environment variables before +# running these jobs; connection strings never belong in a job file. +version: 1 + +jobs: + snapshot-database: + command: convert + description: Read a PostgreSQL database and write a DBML snapshot + inputs: + - format: pgsql + conn_env: SOURCE_DB_URL + output: + format: dbml + path: build/database-snapshot.dbml + overwrite: true + + apply-migrations: + command: scripts-exec + description: Execute migrations from core and tenant folders against PostgreSQL + # All SQL files from these directories are discovered in deterministic order. + script_dirs: + - migrations/core + - migrations/tenant + output: + format: pgsql + conn_env: TARGET_DB_URL + options: + continue_on_error: false + logfile: .relspec/log/apply-migrations.log diff --git a/examples/jobs/relspec.reports.yml b/examples/jobs/relspec.reports.yml new file mode 100644 index 0000000..16b53e8 --- /dev/null +++ b/examples/jobs/relspec.reports.yml @@ -0,0 +1,31 @@ +# Reporting and templating examples. These jobs are discovered together with +# relspec.yml when running `relspec job list` from this directory. +version: 1 + +jobs: + generate-typescript: + command: templ + description: Render one TypeScript file per table from the merged schema + inputs: + - path: schema/core.dbml + format: dbml + - path: schema/tenant.dbml + format: dbml + template: templates/schema.tmpl + mode: table + filename_pattern: "{{.Name}}.ts" + output: + path: "${VST}" + overwrite: true + + compare-schemas: + command: diff + description: Compare the two example schemas and print a summary + inputs: + - path: schema/core.dbml + format: dbml + - path: schema/tenant.dbml + format: dbml + report: + format: summary + logfile: .relspec/log/compare-schemas.log diff --git a/examples/jobs/templates/schema.tmpl b/examples/jobs/templates/schema.tmpl new file mode 100644 index 0000000..2631f68 --- /dev/null +++ b/examples/jobs/templates/schema.tmpl @@ -0,0 +1,6 @@ +// Code generated by RelSpec. DO NOT EDIT. +export interface {{.Name}} { +{{- range values .Table.Columns}} + {{.Name}}: {{.Type}}; +{{- end}} +} diff --git a/pkg/jobs/jobs.go b/pkg/jobs/jobs.go index 696d0c6..c57f771 100644 --- a/pkg/jobs/jobs.go +++ b/pkg/jobs/jobs.go @@ -19,6 +19,7 @@ import ( "fmt" "os" "path/filepath" + "regexp" "sort" "strconv" "strings" @@ -850,6 +851,29 @@ func SafeJoin(root, rel string) (string, error) { return joined, nil } +var envReferencePattern = regexp.MustCompile(`\$\{([A-Za-z_][A-Za-z0-9_]*)\}`) + +// ExpandEnv expands ${NAME} references using the current process environment. +// It deliberately supports only shell-independent variable references; job +// files are never passed through a shell. An unset variable is an error so a +// typo cannot silently turn into a relative path. +func ExpandEnv(value string) (string, error) { + var missing string + expanded := envReferencePattern.ReplaceAllStringFunc(value, func(reference string) string { + name := reference[2 : len(reference)-1] + resolved, ok := os.LookupEnv(name) + if !ok { + missing = name + return reference + } + return resolved + }) + if missing != "" { + return "", fmt.Errorf("environment variable %q referenced by ${%s} is not set", missing, missing) + } + return expanded, nil +} + // deepestExistingAncestor returns p itself if it exists, otherwise the nearest // existing parent directory (falling back to the filesystem root). func deepestExistingAncestor(p string) string { diff --git a/pkg/jobs/jobs_test.go b/pkg/jobs/jobs_test.go index a757251..7f9a1ff 100644 --- a/pkg/jobs/jobs_test.go +++ b/pkg/jobs/jobs_test.go @@ -135,6 +135,21 @@ func TestParseHumanSize(t *testing.T) { } } +func TestExpandEnv(t *testing.T) { + t.Setenv("RELSPEC_TEST_ROOT", "build/output") + got, err := ExpandEnv("${RELSPEC_TEST_ROOT}/schema-${RELSPEC_TEST_ROOT}") + if err != nil { + t.Fatal(err) + } + if want := "build/output/schema-build/output"; got != want { + t.Fatalf("ExpandEnv = %q, want %q", got, want) + } + + if _, err := ExpandEnv("build/${RELSPEC_TEST_MISSING}"); err == nil { + t.Fatal("expected missing environment variable error") + } +} + func TestLoadRejectsDuplicateJobAcrossFiles(t *testing.T) { dir := t.TempDir() a := filepath.Join(dir, "relspec.yml")