diff --git a/README.md b/README.md index 87df12e..5c9a416 100644 --- a/README.md +++ b/README.md @@ -106,6 +106,49 @@ Modes: `database` (default) · `schema` · `table` · `script` Template functions: string utils (`toCamelCase`, `toSnakeCase`, `pluralize`, …), type converters (`sqlToGo`, `sqlToTypeScript`, …), filters, loop helpers, safe access. +### `job` — Declarative job files + +Run named jobs from a `relspec.yml` manifest instead of repeating long command lines. + +```bash +# List jobs discovered in ./relspec.yml and ./relspec..yml (deterministic) +relspec job list + +# Validate and print the plan without running anything +relspec job run build-schema --plan + +# Run a job (and its declared dependencies) +relspec job run build-schema +``` + +```yaml +# relspec.yml +version: 1 +jobs: + build-schema: + command: convert # closed allow-list: convert | merge | scripts-list + description: Merge the DBML sources and emit PostgreSQL DDL + inputs: + - path: schema/core.dbml + format: dbml + - path: schema/tenant.dbml + format: dbml + output: + format: pgsql + path: build/schema.sql + overwrite: true + options: + flatten_schema: false + logfile: .relspec/log/build-schema.log +``` + +The job system is **not** a shell: `command` is a fixed enum, every path is +resolved relative to the job file and may not escape it, and remote database +credentials are referenced by environment-variable name (`conn_env:`) and +redacted from logs. The whole plan — unknown commands/formats, duplicate job +names, missing inputs, path traversal, dependency cycles — is validated before +any job runs. See [docs/JOB_FILES.md](docs/JOB_FILES.md). + ### `edit` — Interactive TUI editor ```bash diff --git a/cmd/relspec/job.go b/cmd/relspec/job.go new file mode 100644 index 0000000..2e1af8d --- /dev/null +++ b/cmd/relspec/job.go @@ -0,0 +1,567 @@ +package main + +import ( + "fmt" + "io" + "os" + "path/filepath" + "sort" + "strings" + "time" + + "github.com/spf13/cobra" + + "git.warky.dev/wdevs/relspecgo/pkg/jobs" + "git.warky.dev/wdevs/relspecgo/pkg/merge" + "git.warky.dev/wdevs/relspecgo/pkg/models" + "git.warky.dev/wdevs/relspecgo/pkg/readers" + "git.warky.dev/wdevs/relspecgo/pkg/readers/sqldir" + wpgsql "git.warky.dev/wdevs/relspecgo/pkg/writers/pgsql" +) + +var ( + jobDir string + jobFiles []string + jobDryRun bool + jobNoDeps bool +) + +var jobCmd = &cobra.Command{ + Use: "job", + Short: "Run declarative RelSpec jobs from job files", + Long: `Run named jobs declared in job files instead of repeating command-line arguments. + +A job file is a YAML manifest (relspec.yml, or relspec..yml for extra +files) describing one or more jobs. Each job names a RelSpec command plus its +inputs, output and options: + + version: 1 + jobs: + build-schema: + command: convert + description: Merge the DBML sources and emit PostgreSQL DDL + inputs: + - path: schema/core.dbml + format: dbml + - path: schema/tenant.dbml + format: dbml + output: + format: pgsql + path: build/schema.sql + overwrite: true + options: + flatten_schema: false + logfile: .relspec/log/build-schema.log + +Rules and guarantees: + - command is a closed allow-list (convert, merge, scripts-list). Arbitrary + shell strings are never executed. + - Every path is relative to the directory holding the job file and may not + escape it. Absolute and home-relative paths are rejected. + - Remote database credentials are referenced by environment-variable name + via conn_env; connection strings are never stored in the manifest and are + redacted from logs and diagnostics. + - Discovery and listing are deterministic. + - The whole plan is validated - unknown commands/formats, duplicate job + names, missing inputs, path traversal, dependency cycles - before any job + runs. Nothing is read, written or executed when validation fails. + - A failed job propagates the underlying non-zero exit status and writes no + success marker.`, +} + +var jobListCmd = &cobra.Command{ + Use: "list", + Short: "List jobs discovered in job files (deterministic order)", + RunE: runJobList, +} + +var jobRunCmd = &cobra.Command{ + Use: "run ", + Short: "Run a named job (and its dependencies) from a job file", + Args: cobra.ExactArgs(1), + RunE: runJobRun, +} + +func init() { + for _, c := range []*cobra.Command{jobListCmd, jobRunCmd} { + c.Flags().StringVar(&jobDir, "dir", ".", "Directory to discover job files in") + c.Flags().StringSliceVar(&jobFiles, "file", nil, "Explicit job file(s) to load (repeatable); disables discovery") + } + jobRunCmd.Flags().BoolVar(&jobDryRun, "dry-run", false, "Validate and print the execution plan without running anything") + jobRunCmd.Flags().BoolVar(&jobDryRun, "plan", false, "Alias for --dry-run") + jobRunCmd.Flags().BoolVar(&jobNoDeps, "no-deps", false, "Run only the named job, skipping its declared dependencies") + + jobCmd.AddCommand(jobListCmd) + jobCmd.AddCommand(jobRunCmd) +} + +// loadJobSet discovers or loads the requested job files and runs full +// validation. The returned Set is safe to plan and execute. +func loadJobSet() (*jobs.Set, error) { + paths := jobFiles + if len(paths) == 0 { + discovered, err := jobs.Discover(jobDir) + if err != nil { + return nil, err + } + paths = discovered + } else { + for i, p := range paths { + if _, err := os.Stat(p); err != nil { + return nil, fmt.Errorf("job file %q: %w", p, err) + } + paths[i] = p + } + } + set, err := jobs.Load(paths) + if err != nil { + return nil, err + } + if err := set.Validate(); err != nil { + return nil, err + } + return set, nil +} + +func runJobList(cmd *cobra.Command, args []string) error { + set, err := loadJobSet() + if err != nil { + return err + } + out := cmd.OutOrStdout() + + fmt.Fprintf(os.Stderr, "\n=== RelSpec Jobs ===\n") + fmt.Fprintf(os.Stderr, "Job files:\n") + for _, f := range set.Files { + fmt.Fprintf(os.Stderr, " - %s\n", f) + } + fmt.Fprintln(os.Stderr) + + names := set.Names() + if len(names) == 0 { + fmt.Fprintln(out, "(no jobs defined)") + return nil + } + + nameW, cmdW, srcW := len("NAME"), len("COMMAND"), len("SOURCE") + for _, n := range names { + j := set.Jobs[n] + nameW = maxInt(nameW, len(n)) + cmdW = maxInt(cmdW, len(j.Command)) + srcW = maxInt(srcW, len(j.SourceFile)) + } + fmt.Fprintf(out, "%-*s %-*s %-*s %s\n", nameW, "NAME", cmdW, "COMMAND", srcW, "SOURCE", "DESCRIPTION") + for _, n := range names { + j := set.Jobs[n] + fmt.Fprintf(out, "%-*s %-*s %-*s %s\n", nameW, n, cmdW, j.Command, srcW, j.SourceFile, j.Description) + } + return nil +} + +func runJobRun(cmd *cobra.Command, args []string) error { + set, err := loadJobSet() + if err != nil { + return err + } + return executeJobPlan(set, args[0], jobDryRun, jobNoDeps, cmd.OutOrStdout()) +} + +// executeJobPlan resolves the plan for name, runs pre-flight checks over +// EVERY job in the plan, and only then executes. When dryRun is set it prints +// the plan and returns without touching any input, output or database. +func executeJobPlan(set *jobs.Set, name string, dryRun, noDeps bool, out io.Writer) error { + plan, err := set.Plan(name, !noDeps) + if err != nil { + return err + } + + // Pre-flight: resolve and check paths, output policy and env vars for the + // whole plan before anything runs. A failure here means no job executes. + resolved := make([]*resolvedJob, len(plan)) + for i, j := range plan { + rj, perr := preflightJob(j) + if perr != nil { + return fmt.Errorf("job %q: %w", j.Name, perr) + } + resolved[i] = rj + } + + if dryRun { + fmt.Fprintf(out, "RelSpec job plan for %q (dry run - nothing executed):\n\n", name) + for i, rj := range resolved { + printResolvedJob(out, i+1, len(resolved), rj) + } + return nil + } + + for _, rj := range resolved { + if err := executeResolvedJob(rj); err != nil { + // Propagate the underlying failure; no success marker is written. + return fmt.Errorf("job %q failed: %w", rj.job.Name, err) + } + } + fmt.Fprintf(os.Stderr, "\n=== Job %q complete ===\n", name) + return nil +} + +// resolvedJob is a job with every manifest path turned into a checked +// absolute filesystem path and every conn_env resolved to its value. +type resolvedJob struct { + job *jobs.Job + root string + inputs []resolvedInput + scriptDirs []string + outputPath string // "" when the output is a database + outputConn string // resolved connection string (secret) + outputConnEnv string + logPath string + secrets []string // resolved secret values to redact from logs +} + +type resolvedInput struct { + format string + path string // "" when the input is a database + conn string // resolved connection string (secret) + connEnv string +} + +func preflightJob(j *jobs.Job) (*resolvedJob, error) { + root := j.Dir() + rj := &resolvedJob{job: j, root: root} + + if j.Logfile != "" { + p, err := jobs.SafeJoin(root, j.Logfile) + if err != nil { + return nil, fmt.Errorf("logfile: %w", err) + } + rj.logPath = p + } + + for i, in := range j.Inputs { + ri := resolvedInput{format: strings.ToLower(in.Format)} + if in.ConnEnv != "" { + v, ok := os.LookupEnv(in.ConnEnv) + if !ok || v == "" { + return nil, fmt.Errorf("input[%d]: environment variable %q (conn_env) is not set", i, in.ConnEnv) + } + ri.conn = v + ri.connEnv = in.ConnEnv + rj.secrets = append(rj.secrets, v) + } else { + p, err := jobs.SafeJoin(root, in.Path) + if err != nil { + return nil, fmt.Errorf("input[%d]: %w", i, err) + } + info, err := os.Stat(p) + if err != nil { + return nil, fmt.Errorf("input[%d]: %s: file not found", i, in.Path) + } + if info.IsDir() { + return nil, fmt.Errorf("input[%d]: %s: is a directory, not a file", i, in.Path) + } + ri.path = p + } + rj.inputs = append(rj.inputs, ri) + } + + for _, d := range j.ScriptDirs { + p, err := jobs.SafeJoin(root, d) + if err != nil { + return nil, fmt.Errorf("script_dir %q: %w", d, err) + } + info, err := os.Stat(p) + if err != nil { + return nil, fmt.Errorf("script_dir %q: not found", d) + } + if !info.IsDir() { + return nil, fmt.Errorf("script_dir %q: not a directory", d) + } + rj.scriptDirs = append(rj.scriptDirs, p) + } + + if j.Output != nil { + if j.Output.ConnEnv != "" { + v, ok := os.LookupEnv(j.Output.ConnEnv) + if !ok || v == "" { + return nil, fmt.Errorf("output: environment variable %q (conn_env) is not set", j.Output.ConnEnv) + } + rj.outputConn = v + rj.outputConnEnv = j.Output.ConnEnv + rj.secrets = append(rj.secrets, v) + } else { + p, err := jobs.SafeJoin(root, j.Output.Path) + if err != nil { + return nil, fmt.Errorf("output: %w", err) + } + if _, err := os.Stat(p); err == nil && !j.Output.Overwrite { + return nil, fmt.Errorf("output %s already exists (set output.overwrite: true to replace it)", j.Output.Path) + } + rj.outputPath = p + } + } + return rj, nil +} + +func printResolvedJob(out io.Writer, n, total int, rj *resolvedJob) { + j := rj.job + fmt.Fprintf(out, "[%d/%d] %s\n", n, total, j.Name) + fmt.Fprintf(out, " command: %s\n", j.Command) + if j.Description != "" { + fmt.Fprintf(out, " description: %s\n", j.Description) + } + fmt.Fprintf(out, " job file: %s\n", j.SourceFile) + for _, ri := range rj.inputs { + if ri.path != "" { + fmt.Fprintf(out, " input: %s (%s)\n", ri.path, ri.format) + } else { + fmt.Fprintf(out, " input: env:%s (%s)\n", ri.connEnv, ri.format) + } + } + for _, d := range rj.scriptDirs { + fmt.Fprintf(out, " script dir: %s\n", d) + } + if rj.outputPath != "" { + fmt.Fprintf(out, " output: %s (%s)\n", rj.outputPath, j.Output.Format) + } else if rj.outputConnEnv != "" { + fmt.Fprintf(out, " output: env:%s (%s)\n", rj.outputConnEnv, j.Output.Format) + } + if rj.logPath != "" { + fmt.Fprintf(out, " logfile: %s\n", rj.logPath) + } + fmt.Fprintln(out) +} + +// executeResolvedJob runs a single already-validated job. +func executeResolvedJob(rj *resolvedJob) (err error) { + lg, closeLog, lerr := newJobLogger(rj.logPath, rj.secrets) + if lerr != nil { + return lerr + } + defer func() { closeLog(err) }() + + lg.logf("=== job %q (%s) started at %s ===", rj.job.Name, rj.job.Command, time.Now().Format(time.RFC3339)) + + switch rj.job.Command { + case jobs.CommandConvert: + err = runConvertJob(rj, lg) + case jobs.CommandMerge: + err = runMergeJob(rj, lg) + case jobs.CommandScriptsList: + err = runScriptsListJob(rj, lg) + default: + err = fmt.Errorf("unsupported command %q", rj.job.Command) + } + if err != nil { + lg.logf("FAILED: %v", err) + } else { + lg.logf("OK") + } + return err +} + +func runConvertJob(rj *resolvedJob, lg *jobLogger) error { + db, err := readJobInputs(rj, lg) + if err != nil { + return err + } + return writeJobOutput(rj, db, lg) +} + +func runMergeJob(rj *resolvedJob, lg *jobLogger) error { + opts := &merge.MergeOptions{ + SkipDomains: rj.job.Options.SkipDomains, + SkipRelations: rj.job.Options.SkipRelations, + SkipEnums: rj.job.Options.SkipEnums, + SkipViews: rj.job.Options.SkipViews, + SkipSequences: rj.job.Options.SkipSequences, + } + var base *models.Database + for i, ri := range rj.inputs { + db, err := readOneJobInput(ri) + if err != nil { + return fmt.Errorf("input[%d]: %w", i, err) + } + if base == nil { + base = db + lg.logf("merge target: %s", inputLabel(ri)) + continue + } + lg.logf("merging: %s", inputLabel(ri)) + merge.MergeDatabases(base, db, opts) + } + base.UpdateDate() + return writeJobOutput(rj, base, lg) +} + +func runScriptsListJob(rj *resolvedJob, lg *jobLogger) error { + type row struct { + priority int + sequence uint + name string + dir string + lines int + } + var rows []row + for _, dir := range rj.scriptDirs { + reader := sqldir.NewReader(&readers.ReaderOptions{ + FilePath: dir, + Metadata: map[string]any{ + "schema_name": valueOr(rj.job.Options.Schema, "public"), + "database_name": "database", + }, + }) + db, err := reader.ReadDatabase() + if err != nil { + return fmt.Errorf("%s: %w", dir, err) + } + if len(db.Schemas) == 0 { + continue + } + for _, s := range db.Schemas[0].Scripts { + lines := strings.Count(s.SQL, "\n") + if len(s.SQL) > 0 && !strings.HasSuffix(s.SQL, "\n") { + lines++ + } + rows = append(rows, row{s.Priority, s.Sequence, s.Name, dir, lines}) + } + } + sort.Slice(rows, func(i, j int) bool { + if rows[i].priority != rows[j].priority { + return rows[i].priority < rows[j].priority + } + if rows[i].sequence != rows[j].sequence { + return rows[i].sequence < rows[j].sequence + } + if rows[i].name != rows[j].name { + return rows[i].name < rows[j].name + } + return rows[i].dir < rows[j].dir + }) + lg.logf("found %d script(s) across %d director(y/ies):", len(rows), len(rj.scriptDirs)) + lg.logf("%-4s %-9s %-9s %-30s %-6s %s", "No.", "Priority", "Sequence", "Name", "Lines", "Directory") + for i, r := range rows { + lg.logf("%-4d %-9d %-9d %-30s %-6d %s", i+1, r.priority, r.sequence, r.name, r.lines, r.dir) + } + return nil +} + +// readJobInputs reads every input and additively merges them into one model. +func readJobInputs(rj *resolvedJob, lg *jobLogger) (*models.Database, error) { + var base *models.Database + for i, ri := range rj.inputs { + db, err := readOneJobInput(ri) + if err != nil { + return nil, fmt.Errorf("input[%d]: %w", i, err) + } + lg.logf("read input: %s", inputLabel(ri)) + if base == nil { + base = db + } else { + merge.MergeDatabases(base, db, &merge.MergeOptions{}) + } + } + if base == nil { + return nil, fmt.Errorf("no inputs produced a database") + } + return base, nil +} + +func readOneJobInput(ri resolvedInput) (*models.Database, error) { + if ri.conn != "" { + return readDatabaseForConvert(ri.format, "", ri.conn) + } + return readDatabaseForConvert(ri.format, ri.path, "") +} + +func inputLabel(ri resolvedInput) string { + if ri.path != "" { + return fmt.Sprintf("%s (%s)", ri.path, ri.format) + } + return fmt.Sprintf("env:%s (%s)", ri.connEnv, ri.format) +} + +// writeJobOutput writes db to the job's output target (file or database). +func writeJobOutput(rj *resolvedJob, db *models.Database, lg *jobLogger) error { + o := rj.job.Options + format := strings.ToLower(rj.job.Output.Format) + + if rj.outputConn != "" { + if format != "pgsql" { + return fmt.Errorf("database output is only supported for pgsql (got %q)", rj.job.Output.Format) + } + lg.logf("writing output to database env:%s", rj.outputConnEnv) + writerOpts := newWriterOptions("", o.Package, o.FlattenSchema, "", "", o.ContinueOnError) + writerOpts.Metadata = map[string]interface{}{"connection_string": rj.outputConn} + return wpgsql.NewWriter(writerOpts).WriteDatabase(db) + } + + if err := os.MkdirAll(filepath.Dir(rj.outputPath), 0o755); err != nil { + return fmt.Errorf("failed to create output directory: %w", err) + } + lg.logf("writing output: %s (%s)", rj.outputPath, format) + return writeDatabase(db, format, rj.outputPath, o.Package, o.Schema, o.FlattenSchema, "", "", o.ContinueOnError, "") +} + +// --- logging + redaction --------------------------------------------------- + +type jobLogger struct { + file io.Writer + secrets []string +} + +// newJobLogger returns a logger that mirrors to stderr and, when path is set, +// to a job logfile. Connection strings and known secret values are redacted +// from everything it writes. +func newJobLogger(path string, secrets []string) (*jobLogger, func(err error), error) { + lg := &jobLogger{secrets: secrets} + if path == "" { + return lg, func(error) {}, nil + } + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + return nil, nil, fmt.Errorf("failed to create log directory: %w", err) + } + f, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0o644) + if err != nil { + return nil, nil, fmt.Errorf("failed to open logfile %q: %w", path, err) + } + lg.file = f + return lg, func(runErr error) { + if runErr != nil { + fmt.Fprintf(f, "%s job ended with error\n", time.Now().Format(time.RFC3339)) + } + _ = f.Close() + }, nil +} + +func (l *jobLogger) logf(format string, args ...interface{}) { + line := l.redact(fmt.Sprintf(format, args...)) + fmt.Fprintf(os.Stderr, " %s\n", line) + if l.file != nil { + fmt.Fprintf(l.file, "%s %s\n", time.Now().Format(time.RFC3339), line) + } +} + +func (l *jobLogger) redact(s string) string { + for _, sec := range l.secrets { + if sec != "" { + s = strings.ReplaceAll(s, sec, "***") + } + } + return maskPassword(s) +} + +// --- small helpers ------------------------------------------------------- + +func maxInt(a, b int) int { + if a > b { + return a + } + return b +} + +func valueOr(v, def string) string { + if v == "" { + return def + } + return v +} diff --git a/cmd/relspec/job_test.go b/cmd/relspec/job_test.go new file mode 100644 index 0000000..2743847 --- /dev/null +++ b/cmd/relspec/job_test.go @@ -0,0 +1,377 @@ +package main + +import ( + "bytes" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/spf13/cobra" + + "git.warky.dev/wdevs/relspecgo/pkg/jobs" +) + +func writeFile(t *testing.T, path, content string) { + t.Helper() + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, []byte(content), 0o644); err != nil { + t.Fatal(err) + } +} + +// jobFixture creates a job-file project with two DBML sources and returns the +// project directory. +func jobFixture(t *testing.T, manifest string) string { + t.Helper() + dir := t.TempDir() + writeFile(t, filepath.Join(dir, "schema", "core.dbml"), "Table users {\n id int [pk]\n name varchar\n}\n") + writeFile(t, filepath.Join(dir, "schema", "tenant.dbml"), "Table posts {\n id int [pk]\n title varchar\n}\n") + writeFile(t, filepath.Join(dir, "relspec.yml"), manifest) + return dir +} + +func mustLoadSet(t *testing.T, files ...string) *jobs.Set { + t.Helper() + set, err := jobs.Load(files) + if err != nil { + t.Fatalf("load: %v", err) + } + if err := set.Validate(); err != nil { + t.Fatalf("validate: %v", err) + } + return set +} + +const convertMergeManifest = `version: 1 +jobs: + build-schema: + command: convert + description: Merge DBML sources to PostgreSQL DDL + inputs: + - path: schema/core.dbml + format: dbml + - path: schema/tenant.dbml + format: dbml + output: + format: pgsql + path: build/schema.sql + overwrite: true + logfile: .relspec/log/build.log +` + +func TestJobRun_ConvertMultiFileMerge(t *testing.T) { + dir := jobFixture(t, convertMergeManifest) + set := mustLoadSet(t, filepath.Join(dir, "relspec.yml")) + + if err := executeJobPlan(set, "build-schema", false, false, &bytes.Buffer{}); err != nil { + t.Fatalf("executeJobPlan: %v", err) + } + + out, err := os.ReadFile(filepath.Join(dir, "build", "schema.sql")) + if err != nil { + t.Fatalf("expected output file: %v", err) + } + sql := string(out) + if !strings.Contains(sql, "users") || !strings.Contains(sql, "posts") { + t.Fatalf("merged output missing tables:\n%s", sql) + } + + logData, err := os.ReadFile(filepath.Join(dir, ".relspec", "log", "build.log")) + if err != nil { + t.Fatalf("expected logfile: %v", err) + } + if !strings.Contains(string(logData), "OK") { + t.Fatalf("logfile missing success marker:\n%s", logData) + } +} + +func TestJobRun_DryRunDoesNotExecute(t *testing.T) { + dir := jobFixture(t, convertMergeManifest) + set := mustLoadSet(t, filepath.Join(dir, "relspec.yml")) + + var buf bytes.Buffer + if err := executeJobPlan(set, "build-schema", true, false, &buf); err != nil { + t.Fatalf("dry run error: %v", err) + } + if !strings.Contains(buf.String(), "dry run") { + t.Fatalf("expected dry-run banner, got: %s", buf.String()) + } + if _, err := os.Stat(filepath.Join(dir, "build", "schema.sql")); !os.IsNotExist(err) { + t.Fatal("dry run must not create the output file") + } + if _, err := os.Stat(filepath.Join(dir, ".relspec", "log", "build.log")); !os.IsNotExist(err) { + t.Fatal("dry run must not create the logfile") + } +} + +func TestJobRun_ValidationFailureNoExecution(t *testing.T) { + badManifest := `version: 1 +jobs: + evil: + command: convert + inputs: + - path: ../../../etc/passwd + format: dbml + output: + format: json + path: build/out.json + logfile: .relspec/evil.log +` + dir := jobFixture(t, badManifest) + if _, err := jobs.Load([]string{filepath.Join(dir, "relspec.yml")}); err != nil { + // structural load ok; validation should reject + t.Fatalf("unexpected load error: %v", err) + } + set, _ := jobs.Load([]string{filepath.Join(dir, "relspec.yml")}) + if err := set.Validate(); err == nil { + t.Fatal("expected validation failure for path traversal") + } + // Nothing should have been produced. + if _, err := os.Stat(filepath.Join(dir, "build")); !os.IsNotExist(err) { + t.Fatal("validation failure must not create output dir") + } + if _, err := os.Stat(filepath.Join(dir, ".relspec")); !os.IsNotExist(err) { + t.Fatal("validation failure must not create logfile dir") + } +} + +func TestJobRun_MissingInputNoExecution(t *testing.T) { + manifest := `version: 1 +jobs: + x: + command: convert + inputs: + - path: schema/does-not-exist.dbml + format: dbml + output: + format: json + path: build/out.json + logfile: .relspec/x.log +` + dir := jobFixture(t, manifest) + set := mustLoadSet(t, filepath.Join(dir, "relspec.yml")) + + err := executeJobPlan(set, "x", false, false, &bytes.Buffer{}) + if err == nil || !strings.Contains(err.Error(), "not found") { + t.Fatalf("expected missing-input error, got %v", err) + } + if _, err := os.Stat(filepath.Join(dir, "build")); !os.IsNotExist(err) { + t.Fatal("missing input must not create output dir") + } + if _, err := os.Stat(filepath.Join(dir, ".relspec")); !os.IsNotExist(err) { + t.Fatal("missing input must not create logfile") + } +} + +func TestJobRun_MissingConnEnvNoExecution(t *testing.T) { + manifest := `version: 1 +jobs: + remote: + command: convert + inputs: + - format: pgsql + conn_env: RELSPEC_TEST_MISSING_CONN + output: + format: json + path: build/out.json + logfile: .relspec/remote.log +` + dir := jobFixture(t, manifest) + os.Unsetenv("RELSPEC_TEST_MISSING_CONN") + set := mustLoadSet(t, filepath.Join(dir, "relspec.yml")) + + err := executeJobPlan(set, "remote", false, false, &bytes.Buffer{}) + if err == nil || !strings.Contains(err.Error(), "conn_env") { + t.Fatalf("expected missing conn_env error, got %v", err) + } + if _, err := os.Stat(filepath.Join(dir, ".relspec")); !os.IsNotExist(err) { + t.Fatal("missing conn_env must not create logfile") + } +} + +func TestJobRun_ExitCodePropagation(t *testing.T) { + // gorm output without options.package makes the underlying writer fail. + manifest := `version: 1 +jobs: + fail: + command: convert + inputs: + - path: schema/core.dbml + format: dbml + output: + format: gorm + path: build/models + overwrite: true + logfile: .relspec/fail.log +` + dir := jobFixture(t, manifest) + set := mustLoadSet(t, filepath.Join(dir, "relspec.yml")) + + err := executeJobPlan(set, "fail", false, false, &bytes.Buffer{}) + if err == nil { + t.Fatal("expected underlying failure to propagate") + } + if !strings.Contains(err.Error(), "job \"fail\" failed") { + t.Fatalf("error should identify the failing job: %v", err) + } + // Logfile records the failure and no misleading success marker. + logData, _ := os.ReadFile(filepath.Join(dir, ".relspec", "fail.log")) + if strings.Contains(string(logData), "\nOK\n") || strings.HasSuffix(strings.TrimSpace(string(logData)), "OK") { + t.Fatalf("failed job must not log OK:\n%s", logData) + } + if !strings.Contains(string(logData), "FAILED") { + t.Fatalf("failed job should log FAILED:\n%s", logData) + } +} + +func TestJobRun_DependencyChainExecutes(t *testing.T) { + manifest := `version: 1 +jobs: + a: + command: convert + inputs: + - path: schema/core.dbml + format: dbml + output: + format: json + path: build/a.json + overwrite: true + b: + command: convert + depends_on: [a] + inputs: + - path: schema/tenant.dbml + format: dbml + output: + format: json + path: build/b.json + overwrite: true +` + dir := jobFixture(t, manifest) + set := mustLoadSet(t, filepath.Join(dir, "relspec.yml")) + + if err := executeJobPlan(set, "b", false, false, &bytes.Buffer{}); err != nil { + t.Fatalf("executeJobPlan: %v", err) + } + for _, f := range []string{"a.json", "b.json"} { + if _, err := os.Stat(filepath.Join(dir, "build", f)); err != nil { + t.Fatalf("expected %s to be produced: %v", f, err) + } + } +} + +func TestJobRun_ScriptsListMultipleDirs(t *testing.T) { + dir := t.TempDir() + writeFile(t, filepath.Join(dir, "migrations", "core", "1_001_create_users.sql"), "CREATE TABLE users();\n") + writeFile(t, filepath.Join(dir, "migrations", "tenant", "1_002_create_posts.sql"), "CREATE TABLE posts();\n") + writeFile(t, filepath.Join(dir, "migrations", "tenant", "2_001_add_index.sql"), "CREATE INDEX x ON posts(id);\n") + manifest := `version: 1 +jobs: + list-all: + command: scripts-list + script_dirs: + - migrations/core + - migrations/tenant + logfile: .relspec/scripts.log +` + writeFile(t, filepath.Join(dir, "relspec.yml"), manifest) + set := mustLoadSet(t, filepath.Join(dir, "relspec.yml")) + + if err := executeJobPlan(set, "list-all", false, false, &bytes.Buffer{}); err != nil { + t.Fatalf("executeJobPlan: %v", err) + } + logData, err := os.ReadFile(filepath.Join(dir, ".relspec", "scripts.log")) + if err != nil { + t.Fatal(err) + } + s := string(logData) + iUsers := strings.Index(s, "create_users") + iPosts := strings.Index(s, "create_posts") + iIndex := strings.Index(s, "add_index") + if iUsers < 0 || iPosts < 0 || iIndex < 0 { + t.Fatalf("expected all scripts listed:\n%s", s) + } + if !(iUsers < iPosts && iPosts < iIndex) { + t.Fatalf("scripts not in priority/sequence order:\n%s", s) + } + if !strings.Contains(s, "found 3 script(s) across 2") { + t.Fatalf("expected multi-directory summary:\n%s", s) + } +} + +func TestJobRun_ConnEnvRedactedInPlan(t *testing.T) { + manifest := `version: 1 +jobs: + remote: + command: convert + inputs: + - format: pgsql + conn_env: RELSPEC_TEST_PLAN_CONN + output: + format: json + path: build/out.json +` + dir := jobFixture(t, manifest) + secret := "postgres://user:supersecret@db.example/app" + t.Setenv("RELSPEC_TEST_PLAN_CONN", secret) + set := mustLoadSet(t, filepath.Join(dir, "relspec.yml")) + + var buf bytes.Buffer + if err := executeJobPlan(set, "remote", true, false, &buf); err != nil { + t.Fatalf("dry run: %v", err) + } + if strings.Contains(buf.String(), "supersecret") || strings.Contains(buf.String(), secret) { + t.Fatalf("plan leaked secret:\n%s", buf.String()) + } + if !strings.Contains(buf.String(), "env:RELSPEC_TEST_PLAN_CONN") { + t.Fatalf("plan should reference the env var name:\n%s", buf.String()) + } +} + +func TestJobLogger_Redaction(t *testing.T) { + lg := &jobLogger{secrets: []string{"topsecret"}} + got := lg.redact("connecting with password topsecret and postgres://u:p@h/db") + if strings.Contains(got, "topsecret") { + t.Fatalf("secret not redacted: %q", got) + } + if !strings.Contains(got, "***") { + t.Fatalf("expected redaction marker: %q", got) + } +} + +func TestJobList_DeterministicOutput(t *testing.T) { + manifest := `version: 1 +jobs: + zebra: + command: convert + inputs: [{path: schema/core.dbml, format: dbml}] + output: {format: json, path: build/z.json} + alpha: + command: convert + inputs: [{path: schema/core.dbml, format: dbml}] + output: {format: json, path: build/a.json} +` + dir := jobFixture(t, manifest) + + run := func() string { + jobDir = dir + jobFiles = nil + cmd := &cobra.Command{} + var buf bytes.Buffer + cmd.SetOut(&buf) + if err := runJobList(cmd, nil); err != nil { + t.Fatalf("runJobList: %v", err) + } + return buf.String() + } + first := run() + if strings.Index(first, "alpha") > strings.Index(first, "zebra") { + t.Fatalf("jobs not sorted:\n%s", first) + } + if first != run() { + t.Fatal("job list output not deterministic") + } +} diff --git a/cmd/relspec/root.go b/cmd/relspec/root.go index df46d41..1e4980c 100644 --- a/cmd/relspec/root.go +++ b/cmd/relspec/root.go @@ -62,6 +62,7 @@ func init() { rootCmd.AddCommand(diffCmd) rootCmd.AddCommand(inspectCmd) rootCmd.AddCommand(scriptsCmd) + rootCmd.AddCommand(jobCmd) rootCmd.AddCommand(assetsCmd) rootCmd.AddCommand(templCmd) rootCmd.AddCommand(editCmd) diff --git a/docs/JOB_FILES.md b/docs/JOB_FILES.md new file mode 100644 index 0000000..e8aca67 --- /dev/null +++ b/docs/JOB_FILES.md @@ -0,0 +1,223 @@ +# RelSpec Job Files + +Job files let you declare named, repeatable RelSpec workflows in YAML and run +them with `relspec job run ` instead of retyping long command lines. + +```bash +relspec job list # deterministic list of discovered jobs +relspec job run build-schema --plan # validate + print plan, execute nothing +relspec job run build-schema # run the job (and its dependencies) +``` + +## Design contract (first release) + +This is the smallest coherent contract that is safe and useful end to end. +Anything not listed under "Supported" is intentionally deferred. + +### Not a shell + +`command` is a **closed allow-list**. There is no field anywhere that accepts a +shell string, an executable path, or arbitrary arguments. Adding a new command +means adding a vetted adapter in the RelSpec source. + +| command | what it does | +|----------------|--------------------------------------------------------------------| +| `convert` | read one or more input schemas, additively merge them, write one output | +| `merge` | like `convert` but requires ≥2 inputs and exposes `skip_*` merge options | +| `scripts-list` | deterministically list SQL scripts across one or more directories | + +Deferred (documented, not implemented here): `scripts` execution against a live +database, `split`, `inspect`, `diff`, `templ`, job-to-job output wiring, +log rotation/retention. Live SQL execution already exists as +`relspec scripts execute`; wiring it into the job runner is a follow-up because +it needs live database credentials and cannot be covered by offline tests. + +### Discovery and precedence + +`relspec job` (no `--file`) scans `--dir` (default `.`) for: + +1. `relspec.yml` / `relspec.yaml` (the default file), then +2. `relspec..yml` / `relspec..yaml` (extra files), + +each group sorted lexically. Order is stable across runs. Use `--file ` +(repeatable) to load explicit files and skip discovery. + +All discovered/selected files are merged into one job namespace. A job name +defined by **more than one file is a hard error** naming both files. YAML maps +already forbid duplicate keys within a single file. + +### Paths + +* Every path (`inputs[].path`, `output.path`, `script_dirs[]`, `logfile`) is + **relative to the directory containing the job file that declared the job**, + not the process working directory. +* Absolute paths, `~`-relative paths and any path that resolves outside the job + file directory (`../`, `a/../../b`, …) are **rejected during validation** — + before anything runs. + +### Credentials + +* Database inputs (`format: pgsql` / `mssql`) and database execution outputs + (`format: pgsql` with `conn_env`) reference an **environment variable name** + via `conn_env:`. The connection string itself is never stored in the + manifest. +* A `conn_env` value that looks like a connection string (contains `:`, `/`, + `@`, `=`, spaces) is rejected. +* Missing/empty environment variables are reported during pre-flight, before + execution. +* Job logs and `--plan` output show `env:`, never the value. Resolved + secret values and anything matching a connection-string password are + redacted (`***`) from the logfile and diagnostics. + +### Validation happens before execution + +`relspec job list` and `relspec job run` both fully validate the selected set +first. Nothing is read, written, connected to, or executed if validation fails. +Checks include: + +* schema `version` (must be `1`), unknown YAML fields rejected +* duplicate job names across files +* unknown / missing `command` +* per-command input/output shape (`convert`/`merge` need inputs + output; + `scripts-list` needs `script_dirs` and forbids inputs/output) +* unknown input/output `format` +* path traversal / absolute / home-relative paths +* `depends_on` targets exist +* dependency cycles (reported as `a -> b -> c -> a`) + +Then, immediately before running, per-job pre-flight resolves paths and checks: + +* every input file exists and is a file +* every `script_dir` exists and is a directory +* every `conn_env` variable is set +* `output.path` does not already exist unless `output.overwrite: true` + +If any pre-flight check fails for **any** job in the plan, **no** job runs. + +### Execution and exit codes + +* `relspec job run ` runs the job's `depends_on` closure first, in + topological order (deterministic), then the job. `--no-deps` runs only the + named job. +* `--dry-run` (alias `--plan`) prints the resolved plan and exits 0 without + touching inputs, outputs or databases. +* A failing job returns the underlying non-zero status (the process exits 1) + and the error names the job. The logfile records `FAILED: `; a + successful job records `OK`. No separate success-marker file is written, so a + failure can never leave a stale "success". + +## Schema reference + +```yaml +version: 1 # required, must be 1 +jobs: + : + command: convert | merge | scripts-list # required + description: "free text" # optional, shown by `job list` + depends_on: [other-job, ...] # optional + inputs: # convert (≥1) / merge (≥2) + - path: relative/file.dbml # file inputs + format: dbml + - format: pgsql # live-connection inputs + conn_env: SOURCE_DB_URL # env var NAME + script_dirs: # scripts-list (≥1) + - migrations/core + - migrations/tenant + output: # convert / merge (required) + format: pgsql + path: build/schema.sql # file output, OR: + conn_env: TARGET_DB_URL # execute against DB (pgsql only) + overwrite: false # default false + options: + flatten_schema: false + schema: public + package: models # for gorm/bun output + continue_on_error: false # pgsql output + skip_relations: false # merge only + skip_enums: false + skip_views: false + skip_domains: false + skip_sequences: false + logfile: .relspec/log/.log # optional; appended to +``` + +### Supported input formats + +`dbml`, `dctx`, `drawdb`, `graphql`, `json`, `yaml`, `gorm`, `bun`, `drizzle`, +`prisma`, `typeorm`, `sqlite` (file, via `path`); `pgsql`, `mssql` +(live, via `conn_env`). + +### Supported output formats + +`dbml`, `dctx`, `drawdb`, `graphql`, `json`, `yaml`, `gorm`, `bun`, `drizzle`, +`prisma`, `typeorm`, `pgsql`, `mssql`, `sqlite` (file, via `path`); `pgsql` also +supports `conn_env` to execute the generated DDL against a live database. + +## Examples + +### Merge many schema files, emit PostgreSQL DDL + +```yaml +version: 1 +jobs: + build-schema: + command: convert + inputs: + - { path: schema/core.dbml, format: dbml } + - { path: schema/billing.dbml, format: dbml } + - { path: schema/tenant.dbml, format: dbml } + output: + format: pgsql + path: build/schema.sql + overwrite: true + logfile: .relspec/log/build-schema.log +``` + +### Multiple script directories + +```yaml +version: 1 +jobs: + migration-order: + command: scripts-list + script_dirs: + - migrations/core + - migrations/tenant + - migrations/reporting + logfile: .relspec/log/migration-order.log +``` + +### Job depending on another job + +```yaml +version: 1 +jobs: + build-schema: + command: convert + inputs: + - { path: schema/core.dbml, format: dbml } + - { path: schema/tenant.dbml, format: dbml } + output: { format: json, path: build/schema.json, overwrite: true } + build-docs: + command: convert + depends_on: [build-schema] + inputs: + - { path: schema/core.dbml, format: dbml } + output: { format: yaml, path: build/schema.yaml, overwrite: true } +``` + +### Reading from a remote database + +```yaml +version: 1 +jobs: + snapshot-prod: + command: convert + inputs: + - format: pgsql + conn_env: PROD_DB_URL # export PROD_DB_URL=postgres://... + output: + format: dbml + path: snapshots/prod.dbml + overwrite: true +``` diff --git a/examples/jobs/.gitignore b/examples/jobs/.gitignore new file mode 100644 index 0000000..9c9128a --- /dev/null +++ b/examples/jobs/.gitignore @@ -0,0 +1,3 @@ +# Generated by `relspec job run` in this example project. +/build/ +/.relspec/ diff --git a/examples/jobs/migrations/core/1_001_create_users.sql b/examples/jobs/migrations/core/1_001_create_users.sql new file mode 100644 index 0000000..5d57733 --- /dev/null +++ b/examples/jobs/migrations/core/1_001_create_users.sql @@ -0,0 +1,4 @@ +CREATE TABLE users ( + id SERIAL PRIMARY KEY, + email VARCHAR NOT NULL UNIQUE +); diff --git a/examples/jobs/migrations/tenant/1_002_create_posts.sql b/examples/jobs/migrations/tenant/1_002_create_posts.sql new file mode 100644 index 0000000..f4f6686 --- /dev/null +++ b/examples/jobs/migrations/tenant/1_002_create_posts.sql @@ -0,0 +1,5 @@ +CREATE TABLE posts ( + id SERIAL PRIMARY KEY, + user_id INT NOT NULL REFERENCES users(id), + title VARCHAR NOT NULL +); diff --git a/examples/jobs/migrations/tenant/2_001_add_posts_user_idx.sql b/examples/jobs/migrations/tenant/2_001_add_posts_user_idx.sql new file mode 100644 index 0000000..9a070b7 --- /dev/null +++ b/examples/jobs/migrations/tenant/2_001_add_posts_user_idx.sql @@ -0,0 +1 @@ +CREATE INDEX posts_user_id_idx ON posts(user_id); diff --git a/examples/jobs/relspec.yml b/examples/jobs/relspec.yml new file mode 100644 index 0000000..64c6452 --- /dev/null +++ b/examples/jobs/relspec.yml @@ -0,0 +1,45 @@ +# Example RelSpec job file. See docs/JOB_FILES.md for the full reference. +# +# cd examples/jobs +# relspec job list +# relspec job run build-schema --plan +# relspec job run build-schema +version: 1 +jobs: + build-schema: + command: convert + description: Merge the DBML sources and emit PostgreSQL DDL + inputs: + - path: schema/core.dbml + format: dbml + - path: schema/tenant.dbml + format: dbml + output: + format: pgsql + path: build/schema.sql + overwrite: true + options: + flatten_schema: false + logfile: .relspec/log/build-schema.log + + build-json: + command: convert + description: Also emit a JSON schema once build-schema succeeds + depends_on: [build-schema] + inputs: + - path: schema/core.dbml + format: dbml + - path: schema/tenant.dbml + format: dbml + output: + format: json + path: build/schema.json + overwrite: true + + migration-order: + command: scripts-list + description: Show the combined execution order across script directories + script_dirs: + - migrations/core + - migrations/tenant + logfile: .relspec/log/migration-order.log diff --git a/examples/jobs/schema/core.dbml b/examples/jobs/schema/core.dbml new file mode 100644 index 0000000..5744c6b --- /dev/null +++ b/examples/jobs/schema/core.dbml @@ -0,0 +1,5 @@ +Table users { + id int [pk, increment] + email varchar [not null, unique] + created_at timestamp +} diff --git a/examples/jobs/schema/tenant.dbml b/examples/jobs/schema/tenant.dbml new file mode 100644 index 0000000..7fd78de --- /dev/null +++ b/examples/jobs/schema/tenant.dbml @@ -0,0 +1,6 @@ +Table posts { + id int [pk, increment] + user_id int [not null, ref: > users.id] + title varchar [not null] + body text +} diff --git a/pkg/jobs/jobs.go b/pkg/jobs/jobs.go new file mode 100644 index 0000000..6e29dab --- /dev/null +++ b/pkg/jobs/jobs.go @@ -0,0 +1,517 @@ +// Package jobs implements RelSpec declarative job files. +// +// A job file is a small YAML manifest that names one or more jobs and, +// for each job, the RelSpec command to run plus its inputs, output and +// options. It lets users run "relspec job run build-schema" instead of +// repeating long command lines. +// +// The job-file system is deliberately NOT a shell: "command" is a closed +// enum of vetted RelSpec workflows, every path is resolved relative to the +// directory holding the job file and may not escape it, and remote database +// credentials are referenced by environment-variable name only - never +// embedded in the manifest. All discovery, parsing and validation in this +// package is side-effect free; nothing here reads input schemas, opens +// database connections or writes output. Execution lives in the CLI layer +// and only runs after Validate and the caller's pre-flight checks pass. +package jobs + +import ( + "fmt" + "os" + "path/filepath" + "sort" + "strings" + + "gopkg.in/yaml.v3" +) + +// SchemaVersion is the only job-file schema version this build understands. +const SchemaVersion = 1 + +// Command names are a closed allow-list. Arbitrary strings are rejected. +const ( + CommandConvert = "convert" // read one or more schema files, optionally merge, write one output + CommandMerge = "merge" // additive merge of two or more schema files into one output + CommandScriptsList = "scripts-list" // deterministically list SQL scripts across one or more directories +) + +// SupportedCommands lists every accepted command, in help order. +var SupportedCommands = []string{CommandConvert, CommandMerge, CommandScriptsList} + +// readerFormats are the file-based input formats a job may declare (path). +var readerFormats = map[string]bool{ + "dbml": true, "dctx": true, "drawdb": true, "graphql": true, "json": true, + "yaml": true, "gorm": true, "bun": true, "drizzle": true, "prisma": true, + "typeorm": true, "sqlite": true, +} + +// inputDBFormats are input formats that can only come from a live connection, +// referenced by conn_env. +var inputDBFormats = map[string]bool{"pgsql": true, "mssql": true} + +// writerFormats are the output formats a job may declare. +var writerFormats = map[string]bool{ + "dbml": true, "dctx": true, "drawdb": true, "graphql": true, "json": true, + "yaml": true, "gorm": true, "bun": true, "drizzle": true, "prisma": true, + "typeorm": true, "pgsql": true, "mssql": true, "sqlite": true, +} + +// execOutputFormats are output formats for which conn_env (execute against a +// live database) is supported instead of writing a file. +var execOutputFormats = map[string]bool{"pgsql": true} + +// File is the on-disk shape of a single job file. +type File struct { + Version int `yaml:"version"` + Jobs map[string]*Job `yaml:"jobs"` +} + +// Job is one named job within a job file. +type Job struct { + // Name and SourceFile are populated by Load, not parsed from YAML. + Name string `yaml:"-"` + SourceFile string `yaml:"-"` + + Command string `yaml:"command"` + Description string `yaml:"description"` + DependsOn []string `yaml:"depends_on"` + Inputs []Input `yaml:"inputs"` + ScriptDirs []string `yaml:"script_dirs"` + Output *Output `yaml:"output"` + Options Options `yaml:"options"` + Logfile string `yaml:"logfile"` +} + +// Input is one declared input schema. +type Input struct { + Path string `yaml:"path"` + // Format is the RelSpec reader format (dbml, json, yaml, pgsql, ...). + Format string `yaml:"format"` + // ConnEnv is the NAME of an environment variable holding a connection + // string, used with database formats. The value is never stored here. + ConnEnv string `yaml:"conn_env"` +} + +// Output is the declared output target. +type Output struct { + Format string `yaml:"format"` + Path string `yaml:"path"` + ConnEnv string `yaml:"conn_env"` + Overwrite bool `yaml:"overwrite"` +} + +// Options carries the subset of command flags a job file may set. +type Options struct { + FlattenSchema bool `yaml:"flatten_schema"` + Schema string `yaml:"schema"` + Package string `yaml:"package"` + ContinueOnError bool `yaml:"continue_on_error"` + SkipRelations bool `yaml:"skip_relations"` + SkipEnums bool `yaml:"skip_enums"` + SkipViews bool `yaml:"skip_views"` + SkipDomains bool `yaml:"skip_domains"` + SkipSequences bool `yaml:"skip_sequences"` +} + +// Dir returns the directory that a job's relative paths resolve against: +// the directory containing the job file that declared it. +func (j *Job) Dir() string { return filepath.Dir(j.SourceFile) } + +// Set is the merged view of all discovered/selected job files. +type Set struct { + // Files is the sorted list of job files that contributed jobs. + Files []string + // Jobs is keyed by job name. + Jobs map[string]*Job +} + +// Names returns all job names in deterministic (sorted) order. +func (s *Set) Names() []string { + names := make([]string, 0, len(s.Jobs)) + for n := range s.Jobs { + names = append(names, n) + } + sort.Strings(names) + return names +} + +// Discover returns the job files in dir in deterministic order. The default +// file "relspec.yml"/"relspec.yaml" sorts first, followed by named files +// "relspec..yml"/"relspec..yaml" in lexical order. +func Discover(dir string) ([]string, error) { + if dir == "" { + dir = "." + } + entries, err := os.ReadDir(dir) + if err != nil { + return nil, fmt.Errorf("failed to read directory %q: %w", dir, err) + } + var defaults, named []string + for _, e := range entries { + if e.IsDir() { + continue + } + name := e.Name() + if !isJobFileName(name) { + continue + } + full := filepath.Join(dir, name) + if name == "relspec.yml" || name == "relspec.yaml" { + defaults = append(defaults, full) + } else { + named = append(named, full) + } + } + sort.Strings(defaults) + sort.Strings(named) + return append(defaults, named...), nil +} + +func isJobFileName(name string) bool { + for _, ext := range []string{".yml", ".yaml"} { + if name == "relspec"+ext { + return true + } + if strings.HasPrefix(name, "relspec.") && strings.HasSuffix(name, ext) { + return true + } + } + return false +} + +// Load parses every path, rejects unknown fields and unsupported versions, +// and merges all jobs into one Set. A job name defined by more than one file +// is a hard error. Load performs structural checks only; call Validate for +// full semantic validation. +func Load(paths []string) (*Set, error) { + if len(paths) == 0 { + return nil, fmt.Errorf("no job files found (looked for relspec.yml / relspec..yml)") + } + set := &Set{Jobs: map[string]*Job{}} + origin := map[string]string{} // job name -> first file that defined it + + for _, path := range paths { + data, err := os.ReadFile(path) + if err != nil { + return nil, fmt.Errorf("failed to read job file %q: %w", path, err) + } + dec := yaml.NewDecoder(strings.NewReader(string(data))) + dec.KnownFields(true) + var f File + if err := dec.Decode(&f); err != nil { + return nil, fmt.Errorf("invalid job file %q: %w", path, err) + } + if f.Version != SchemaVersion { + return nil, fmt.Errorf("job file %q: unsupported version %d (expected %d)", path, f.Version, SchemaVersion) + } + if len(f.Jobs) == 0 { + return nil, fmt.Errorf("job file %q: no jobs defined", path) + } + for name, job := range f.Jobs { + if job == nil { + return nil, fmt.Errorf("job file %q: job %q is empty", path, name) + } + if prev, dup := origin[name]; dup { + return nil, fmt.Errorf("duplicate job %q defined in both %q and %q", name, prev, path) + } + job.Name = name + job.SourceFile = path + origin[name] = path + set.Jobs[name] = job + } + set.Files = append(set.Files, path) + } + return set, nil +} + +// Validate runs full semantic validation over the whole set and returns a +// single error describing every problem found. It never touches the +// filesystem beyond what Load already read; existence of input files and +// environment variables is checked by the caller immediately before +// execution. +func (s *Set) Validate() error { + var errs []string + for _, name := range s.Names() { + for _, msg := range s.Jobs[name].validate() { + errs = append(errs, fmt.Sprintf("job %q: %s", name, msg)) + } + } + // Dependency references + cycles. + for _, name := range s.Names() { + for _, dep := range s.Jobs[name].DependsOn { + if _, ok := s.Jobs[dep]; !ok { + errs = append(errs, fmt.Sprintf("job %q: depends_on unknown job %q", name, dep)) + } + } + } + if cycle := s.findCycle(); cycle != "" { + errs = append(errs, fmt.Sprintf("dependency cycle detected: %s", cycle)) + } + if len(errs) > 0 { + sort.Strings(errs) + return fmt.Errorf("job file validation failed:\n - %s", strings.Join(errs, "\n - ")) + } + return nil +} + +func (j *Job) validate() []string { + var e []string + + switch j.Command { + case CommandConvert, CommandMerge, CommandScriptsList: + case "": + e = append(e, "missing command") + return e + default: + e = append(e, fmt.Sprintf("unsupported command %q (supported: %s)", j.Command, strings.Join(SupportedCommands, ", "))) + return e + } + + // Path safety for every declared path. + checkPath := func(label, p string) { + if p == "" { + return + } + if err := checkRelPath(p); err != nil { + e = append(e, fmt.Sprintf("%s %q: %v", label, p, err)) + } + } + checkPath("logfile", j.Logfile) + for _, in := range j.Inputs { + checkPath("input path", in.Path) + } + for _, d := range j.ScriptDirs { + checkPath("script_dir", d) + } + if j.Output != nil { + checkPath("output path", j.Output.Path) + } + + switch j.Command { + case CommandConvert, CommandMerge: + minInputs := 1 + if j.Command == CommandMerge { + minInputs = 2 + } + if len(j.Inputs) < minInputs { + e = append(e, fmt.Sprintf("command %q requires at least %d input(s)", j.Command, minInputs)) + } + for i, in := range j.Inputs { + e = append(e, validateInput(i, in)...) + } + if len(j.ScriptDirs) > 0 { + e = append(e, fmt.Sprintf("script_dirs is not valid for command %q", j.Command)) + } + if j.Output == nil { + e = append(e, "missing output") + } else { + e = append(e, validateOutput(*j.Output)...) + } + case CommandScriptsList: + if len(j.ScriptDirs) == 0 { + e = append(e, "command \"scripts-list\" requires at least one script_dir") + } + if len(j.Inputs) > 0 { + e = append(e, "inputs is not valid for command \"scripts-list\"") + } + if j.Output != nil { + e = append(e, "output is not valid for command \"scripts-list\"") + } + } + return e +} + +func validateInput(i int, in Input) []string { + var e []string + if in.Format == "" { + e = append(e, fmt.Sprintf("input[%d]: missing format", i)) + return e + } + f := strings.ToLower(in.Format) + switch { + case inputDBFormats[f]: + if in.ConnEnv == "" { + e = append(e, fmt.Sprintf("input[%d]: format %q requires conn_env (an environment variable name)", i, in.Format)) + } + if in.Path != "" { + e = append(e, fmt.Sprintf("input[%d]: format %q takes conn_env, not path", i, in.Format)) + } + case readerFormats[f]: + if in.Path == "" { + e = append(e, fmt.Sprintf("input[%d]: missing path", i)) + } + if in.ConnEnv != "" { + e = append(e, fmt.Sprintf("input[%d]: format %q does not use conn_env", i, in.Format)) + } + default: + e = append(e, fmt.Sprintf("input[%d]: unsupported input format %q", i, in.Format)) + } + if looksLikeSecret(in.ConnEnv) { + e = append(e, fmt.Sprintf("input[%d]: conn_env must be an environment variable name, not a connection string", i)) + } + return e +} + +func validateOutput(o Output) []string { + var e []string + if o.Format == "" { + e = append(e, "output: missing format") + return e + } + f := strings.ToLower(o.Format) + if !writerFormats[f] { + e = append(e, fmt.Sprintf("output: unsupported output format %q", o.Format)) + return e + } + if o.ConnEnv != "" { + if !execOutputFormats[f] { + e = append(e, fmt.Sprintf("output: conn_env (live database execution) is not supported for format %q", o.Format)) + } + if o.Path != "" { + e = append(e, "output: set either path or conn_env, not both") + } + } else if o.Path == "" { + e = append(e, "output: missing path") + } + if looksLikeSecret(o.ConnEnv) { + e = append(e, "output: conn_env must be an environment variable name, not a connection string") + } + return e +} + +// looksLikeSecret reports whether s looks like a connection string rather +// than a bare environment-variable name. +func looksLikeSecret(s string) bool { + if s == "" { + return false + } + return strings.ContainsAny(s, ":/@ =") || strings.Contains(s, "//") +} + +// checkRelPath rejects absolute paths and any path that escapes its root. +func checkRelPath(p string) error { + if p == "" { + return fmt.Errorf("empty path") + } + if filepath.IsAbs(p) { + return fmt.Errorf("absolute paths are not allowed; use a path relative to the job file") + } + if strings.HasPrefix(p, "~") { + return fmt.Errorf("home-relative paths are not allowed") + } + clean := filepath.ToSlash(filepath.Clean(p)) + if clean == ".." || strings.HasPrefix(clean, "../") { + return fmt.Errorf("path escapes the job file directory") + } + return nil +} + +// SafeJoin resolves rel against root and guarantees the result stays inside +// root. It is the single choke point for turning a manifest path into a +// filesystem path. +func SafeJoin(root, rel string) (string, error) { + if err := checkRelPath(rel); err != nil { + return "", err + } + absRoot, err := filepath.Abs(root) + if err != nil { + return "", err + } + joined := filepath.Join(absRoot, rel) + rp, err := filepath.Rel(absRoot, joined) + if err != nil { + return "", err + } + if rp == ".." || strings.HasPrefix(rp, ".."+string(filepath.Separator)) { + return "", fmt.Errorf("path %q escapes the job file directory", rel) + } + return joined, nil +} + +// Plan returns the jobs to execute for name in dependency order. When +// includeDeps is false only the named job is returned (its declared +// dependencies are still validated to exist and be acyclic by Validate). +func (s *Set) Plan(name string, includeDeps bool) ([]*Job, error) { + root, ok := s.Jobs[name] + if !ok { + return nil, fmt.Errorf("unknown job %q (known: %s)", name, strings.Join(s.Names(), ", ")) + } + if !includeDeps { + return []*Job{root}, nil + } + var order []*Job + visited := map[string]bool{} + inProgress := map[string]bool{} + var visit func(n string) error + visit = func(n string) error { + if visited[n] { + return nil + } + if inProgress[n] { + return fmt.Errorf("dependency cycle at job %q", n) + } + inProgress[n] = true + j := s.Jobs[n] + deps := append([]string(nil), j.DependsOn...) + sort.Strings(deps) + for _, d := range deps { + if _, ok := s.Jobs[d]; !ok { + return fmt.Errorf("job %q depends on unknown job %q", n, d) + } + if err := visit(d); err != nil { + return err + } + } + inProgress[n] = false + visited[n] = true + order = append(order, j) + return nil + } + if err := visit(name); err != nil { + return nil, err + } + return order, nil +} + +// findCycle returns a human-readable cycle path, or "" if the graph is acyclic. +func (s *Set) findCycle() string { + color := map[string]int{} // 0 unvisited, 1 in progress, 2 done + var stack []string + var dfs func(n string) []string + dfs = func(n string) []string { + color[n] = 1 + stack = append(stack, n) + deps := append([]string(nil), s.Jobs[n].DependsOn...) + sort.Strings(deps) + for _, d := range deps { + if _, ok := s.Jobs[d]; !ok { + continue + } + switch color[d] { + case 0: + if c := dfs(d); c != nil { + return c + } + case 1: + // Found a back edge; build the cycle slice. + for i, x := range stack { + if x == d { + return append(append([]string(nil), stack[i:]...), d) + } + } + return []string{d, d} + } + } + stack = stack[:len(stack)-1] + color[n] = 2 + return nil + } + for _, n := range s.Names() { + if color[n] == 0 { + if c := dfs(n); c != nil { + return strings.Join(c, " -> ") + } + } + } + return "" +} diff --git a/pkg/jobs/jobs_test.go b/pkg/jobs/jobs_test.go new file mode 100644 index 0000000..6f50dfe --- /dev/null +++ b/pkg/jobs/jobs_test.go @@ -0,0 +1,251 @@ +package jobs + +import ( + "os" + "path/filepath" + "strings" + "testing" +) + +func write(t *testing.T, path, content string) { + t.Helper() + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, []byte(content), 0o644); err != nil { + t.Fatal(err) + } +} + +func TestDiscoverDeterministicOrder(t *testing.T) { + dir := t.TempDir() + for _, n := range []string{ + "relspec.yml", "relspec.zeta.yml", "relspec.alpha.yaml", + "relspec.beta.yml", "notes.yml", "relspec.txt", + } { + write(t, filepath.Join(dir, n), "version: 1\njobs: {}\n") + } + got, err := Discover(dir) + if err != nil { + t.Fatal(err) + } + var bases []string + for _, p := range got { + bases = append(bases, filepath.Base(p)) + } + want := []string{"relspec.yml", "relspec.alpha.yaml", "relspec.beta.yml", "relspec.zeta.yml"} + if strings.Join(bases, ",") != strings.Join(want, ",") { + t.Fatalf("discover order = %v, want %v", bases, want) + } + + // Second call must return the identical order. + got2, _ := Discover(dir) + for i := range got { + if got[i] != got2[i] { + t.Fatalf("discover not deterministic: %v vs %v", got, got2) + } + } +} + +func TestLoadRejectsUnknownFields(t *testing.T) { + dir := t.TempDir() + p := filepath.Join(dir, "relspec.yml") + write(t, p, "version: 1\njobs:\n a:\n command: convert\n bogus: true\n") + if _, err := Load([]string{p}); err == nil { + t.Fatal("expected error for unknown field") + } +} + +func TestLoadRejectsBadVersion(t *testing.T) { + dir := t.TempDir() + p := filepath.Join(dir, "relspec.yml") + write(t, p, "version: 2\njobs:\n a:\n command: convert\n") + _, err := Load([]string{p}) + if err == nil || !strings.Contains(err.Error(), "unsupported version") { + t.Fatalf("expected unsupported version error, got %v", err) + } +} + +func TestLoadRejectsDuplicateJobAcrossFiles(t *testing.T) { + dir := t.TempDir() + a := filepath.Join(dir, "relspec.yml") + b := filepath.Join(dir, "relspec.extra.yml") + write(t, a, jobFileConvert("build")) + write(t, b, jobFileConvert("build")) + _, err := Load([]string{a, b}) + if err == nil || !strings.Contains(err.Error(), "duplicate job") { + t.Fatalf("expected duplicate job error, got %v", err) + } +} + +func jobFileConvert(name string) string { + return "version: 1\njobs:\n " + name + ":\n command: convert\n" + + " inputs:\n - path: a.dbml\n format: dbml\n" + + " output:\n format: json\n path: out.json\n" +} + +func loadOne(t *testing.T, content string) *Set { + t.Helper() + dir := t.TempDir() + p := filepath.Join(dir, "relspec.yml") + write(t, p, content) + set, err := Load([]string{p}) + if err != nil { + t.Fatalf("load: %v", err) + } + return set +} + +func TestValidateUnknownCommand(t *testing.T) { + set := loadOne(t, "version: 1\njobs:\n x:\n command: rm-rf\n") + err := set.Validate() + if err == nil || !strings.Contains(err.Error(), "unsupported command") { + t.Fatalf("want unsupported command, got %v", err) + } +} + +func TestValidateShellStringCommandRejected(t *testing.T) { + set := loadOne(t, "version: 1\njobs:\n x:\n command: \"bash -c 'echo hi'\"\n") + if err := set.Validate(); err == nil { + t.Fatal("expected arbitrary shell command to be rejected") + } +} + +func TestValidateMissingInputs(t *testing.T) { + set := loadOne(t, "version: 1\njobs:\n x:\n command: convert\n output:\n format: json\n path: o.json\n") + if err := set.Validate(); err == nil || !strings.Contains(err.Error(), "at least 1 input") { + t.Fatalf("want missing input error, got %v", err) + } +} + +func TestValidateUnknownFormat(t *testing.T) { + set := loadOne(t, "version: 1\njobs:\n x:\n command: convert\n"+ + " inputs:\n - path: a.xyz\n format: xyz\n"+ + " output:\n format: json\n path: o.json\n") + if err := set.Validate(); err == nil || !strings.Contains(err.Error(), "unsupported input format") { + t.Fatalf("want unsupported input format, got %v", err) + } +} + +func TestValidatePathTraversalRejected(t *testing.T) { + cases := []string{"../secret.dbml", "/etc/passwd", "~/x.dbml", "a/../../b.dbml"} + for _, bad := range cases { + set := loadOne(t, "version: 1\njobs:\n x:\n command: convert\n"+ + " inputs:\n - path: \""+bad+"\"\n format: dbml\n"+ + " output:\n format: json\n path: o.json\n") + if err := set.Validate(); err == nil { + t.Fatalf("path %q: expected rejection", bad) + } + } +} + +func TestValidateOutputTraversalRejected(t *testing.T) { + set := loadOne(t, "version: 1\njobs:\n x:\n command: convert\n"+ + " inputs:\n - path: a.dbml\n format: dbml\n"+ + " output:\n format: json\n path: ../../evil.json\n") + if err := set.Validate(); err == nil { + t.Fatal("expected output path traversal rejection") + } +} + +func TestValidateConnEnvMustBeName(t *testing.T) { + set := loadOne(t, "version: 1\njobs:\n x:\n command: convert\n"+ + " inputs:\n - format: pgsql\n conn_env: \"postgres://u:p@h/db\"\n"+ + " output:\n format: json\n path: o.json\n") + if err := set.Validate(); err == nil || !strings.Contains(err.Error(), "environment variable name") { + t.Fatalf("want conn_env name error, got %v", err) + } +} + +func TestValidateDependsOnUnknown(t *testing.T) { + set := loadOne(t, "version: 1\njobs:\n x:\n command: convert\n depends_on: [nope]\n"+ + " inputs:\n - path: a.dbml\n format: dbml\n"+ + " output:\n format: json\n path: o.json\n") + if err := set.Validate(); err == nil || !strings.Contains(err.Error(), "unknown job") { + t.Fatalf("want unknown dependency error, got %v", err) + } +} + +func TestValidateDependencyCycle(t *testing.T) { + content := "version: 1\njobs:\n" + + jobBlock("a", "b") + jobBlock("b", "c") + jobBlock("c", "a") + set := loadOne(t, content) + err := set.Validate() + if err == nil || !strings.Contains(err.Error(), "cycle") { + t.Fatalf("want cycle error, got %v", err) + } +} + +func jobBlock(name, dep string) string { + return " " + name + ":\n command: convert\n depends_on: [" + dep + "]\n" + + " inputs:\n - path: a.dbml\n format: dbml\n" + + " output:\n format: json\n path: " + name + ".json\n" +} + +func TestPlanTopologicalOrder(t *testing.T) { + content := "version: 1\njobs:\n" + + " base:\n command: convert\n inputs:\n - path: a.dbml\n format: dbml\n output:\n format: json\n path: base.json\n" + + " mid:\n command: convert\n depends_on: [base]\n inputs:\n - path: a.dbml\n format: dbml\n output:\n format: json\n path: mid.json\n" + + " top:\n command: convert\n depends_on: [mid]\n inputs:\n - path: a.dbml\n format: dbml\n output:\n format: json\n path: top.json\n" + set := loadOne(t, content) + if err := set.Validate(); err != nil { + t.Fatalf("validate: %v", err) + } + plan, err := set.Plan("top", true) + if err != nil { + t.Fatal(err) + } + var order []string + for _, j := range plan { + order = append(order, j.Name) + } + if strings.Join(order, ",") != "base,mid,top" { + t.Fatalf("plan order = %v, want [base mid top]", order) + } + + solo, err := set.Plan("top", false) + if err != nil { + t.Fatal(err) + } + if len(solo) != 1 || solo[0].Name != "top" { + t.Fatalf("no-deps plan = %v, want [top]", solo) + } +} + +func TestSafeJoinStaysInsideRoot(t *testing.T) { + root := t.TempDir() + if _, err := SafeJoin(root, "sub/dir/file.sql"); err != nil { + t.Fatalf("expected ok, got %v", err) + } + if _, err := SafeJoin(root, "../escape"); err == nil { + t.Fatal("expected escape rejection") + } + if _, err := SafeJoin(root, "/abs"); err == nil { + t.Fatal("expected absolute rejection") + } +} + +func TestShippedExampleIsValid(t *testing.T) { + path := filepath.Join("..", "..", "examples", "jobs", "relspec.yml") + set, err := Load([]string{path}) + if err != nil { + t.Fatalf("load example: %v", err) + } + if err := set.Validate(); err != nil { + t.Fatalf("example manifest failed validation: %v", err) + } + if _, err := set.Plan("build-json", true); err != nil { + t.Fatalf("plan example: %v", err) + } +} + +func TestScriptsListValidation(t *testing.T) { + set := loadOne(t, "version: 1\njobs:\n s:\n command: scripts-list\n") + if err := set.Validate(); err == nil || !strings.Contains(err.Error(), "script_dir") { + t.Fatalf("want script_dir required error, got %v", err) + } + set = loadOne(t, "version: 1\njobs:\n s:\n command: scripts-list\n script_dirs: [migrations, extra]\n") + if err := set.Validate(); err != nil { + t.Fatalf("expected valid scripts-list job, got %v", err) + } +}