feat(job): complete deferred job-file features

Implements the remaining items from issue #20: - version is now forward-permissive: any value >= 1 is accepted; a newer-than-known version loads best-effort (unknown fields ignored, warning printed) instead of hard-failing on "must be 1" - from_job input reference: `inputs: [{ from_job: <job> }]` resolves to that job's single-file output + format and implies a dependency edge; combined depends_on + from_job graph gets topological ordering and cycle detection - logfile size-rotation, on by default (5MB, keep 3), overridable per job (log_max_size / log_keep) or file-wide via a top-level defaults block - new commands: split (schema/table subsetting via select:), inspect (rule validation -> markdown/json report, fails job on enforced-rule errors), diff (compare exactly two schemas, never fails), scripts-exec (run SQL script dirs against a live PostgreSQL database) - atomic single-file output/report writes (temp file + rename) - symlink-escape hardening in SafeJoin via EvalSymlinks preflight Updates docs/JOB_FILES.md and examples/jobs/relspec.yml accordingly.
2026-09-08 14:32:07 +02:00
parent f968e3d4a6
commit 84a6b31873
7 changed files with 1434 additions and 62 deletions
@@ -20,24 +20,52 @@ import (
 	"os"
 	"path/filepath"
 	"sort"
+	"strconv"
 	"strings"

 	"gopkg.in/yaml.v3"
 )

-// SchemaVersion is the only job-file schema version this build understands.
-const SchemaVersion = 1
+// CurrentSchemaVersion is the highest job-file schema version this build was
+// written for. MinSchemaVersion is the oldest it still accepts. A file that
+// declares a version in between loads normally; a newer version loads
+// best-effort with a warning (see Load); an older-than-minimum version is a
+// hard error.
+const (
+	CurrentSchemaVersion = 1
+	MinSchemaVersion     = 1
+)
+
+// Built-in logfile rotation policy, used when neither the job nor its file's
+// defaults block sets one.
+const (
+	defaultLogMaxSizeBytes int64 = 5 << 20 // 5 MiB
+	defaultLogKeep               = 3
+)

 // Command names are a closed allow-list. Arbitrary strings are rejected.
 const (
 	CommandConvert     = "convert"      // read one or more schema files, optionally merge, write one output
 	CommandMerge       = "merge"        // additive merge of two or more schema files into one output
 	CommandScriptsList = "scripts-list" // deterministically list SQL scripts across one or more directories
+	CommandScriptsExec = "scripts-exec" // execute SQL scripts across one or more directories against a live database
 	CommandTempl       = "templ"        // apply a custom Go text template to one or more schemas
+	CommandSplit       = "split"        // extract selected schemas/tables into a separate output
+	CommandInspect     = "inspect"      // validate one or more schemas against rules and write a report
+	CommandDiff        = "diff"         // compare exactly two schemas and write a differences report
 )

 // SupportedCommands lists every accepted command, in help order.
-var SupportedCommands = []string{CommandConvert, CommandMerge, CommandScriptsList, CommandTempl}
+var SupportedCommands = []string{
+	CommandConvert, CommandMerge, CommandScriptsList, CommandScriptsExec,
+	CommandTempl, CommandSplit, CommandInspect, CommandDiff,
+}
+
+// producerCommands are commands whose output is a schema file that another job
+// may consume via from_job.
+var producerCommands = map[string]bool{
+	CommandConvert: true, CommandMerge: true, CommandSplit: true,
+}

 // readerFormats are the file-based input formats a job may declare (path).
 var readerFormats = map[string]bool{
@@ -61,10 +89,41 @@ var writerFormats = map[string]bool{
 // live database) is supported instead of writing a file.
 var execOutputFormats = map[string]bool{"pgsql": true}

+// singleFileFormats are output formats that emit exactly one file (as opposed
+// to a directory of files). Only these are eligible for atomic temp+rename
+// writes and for being consumed by another job via from_job.
+var singleFileFormats = map[string]bool{
+	"json": true, "yaml": true, "dbml": true, "dctx": true, "drawdb": true,
+	"graphql": true, "pgsql": true, "mssql": true, "sqlite": true,
+}
+
+// SingleFileOutputFormat reports whether format writes exactly one file.
+func SingleFileOutputFormat(format string) bool {
+	return singleFileFormats[strings.ToLower(format)]
+}
+
+// diffReportFormats and inspectReportFormats are the report.format values
+// accepted by the diff and inspect commands respectively.
+var (
+	diffReportFormats    = map[string]bool{"summary": true, "json": true, "html": true}
+	inspectReportFormats = map[string]bool{"markdown": true, "json": true}
+)
+
 // File is the on-disk shape of a single job file.
 type File struct {
-	Version int             `yaml:"version"`
-	Jobs    map[string]*Job `yaml:"jobs"`
+	Version  int             `yaml:"version"`
+	Defaults *Defaults       `yaml:"defaults"`
+	Jobs     map[string]*Job `yaml:"jobs"`
+}
+
+// Defaults carries file-wide settings that individual jobs may override.
+type Defaults struct {
+	// LogMaxSize is a human-readable size ("5MB", "512KB", "1GB"). Empty
+	// means "use the built-in default".
+	LogMaxSize string `yaml:"log_max_size"`
+	// LogKeep is how many rotated logfiles to retain. Zero means "use the
+	// built-in default".
+	LogKeep int `yaml:"log_keep"`
 }

 // Job is one named job within a job file.
@@ -72,6 +131,9 @@ type Job struct {
 	// Name and SourceFile are populated by Load, not parsed from YAML.
 	Name       string `yaml:"-"`
 	SourceFile string `yaml:"-"`
+	// fileDefaults is the Defaults block of the file that declared this job,
+	// captured by Load. nil when the file had none.
+	fileDefaults *Defaults `yaml:"-"`

 	Command         string   `yaml:"command"`
 	Description     string   `yaml:"description"`
@@ -82,8 +144,13 @@ type Job struct {
 	Mode            string   `yaml:"mode"`
 	FilenamePattern string   `yaml:"filename_pattern"`
 	Output          *Output  `yaml:"output"`
+	Rules           string   `yaml:"rules"`
+	Report          *Report  `yaml:"report"`
+	Select          *Select  `yaml:"select"`
 	Options         Options  `yaml:"options"`
 	Logfile         string   `yaml:"logfile"`
+	LogMaxSize      string   `yaml:"log_max_size"`
+	LogKeep         *int     `yaml:"log_keep"`
 }

 // Input is one declared input schema.
@@ -94,6 +161,10 @@ type Input struct {
 	// ConnEnv is the NAME of an environment variable holding a connection
 	// string, used with database formats. The value is never stored here.
 	ConnEnv string `yaml:"conn_env"`
+	// FromJob names another job in the set whose file output is used as this
+	// input. It implies a dependency on that job. Path/Format/ConnEnv must be
+	// empty when FromJob is set; the format is inherited from the producer.
+	FromJob string `yaml:"from_job"`
 }

 // Output is the declared output target.
@@ -104,6 +175,105 @@ type Output struct {
 	Overwrite bool   `yaml:"overwrite"`
 }

+// Report is the output target for the inspect and diff commands.
+type Report struct {
+	// Format is the report format: diff accepts summary|json|html, inspect
+	// accepts markdown|json. Empty means the command's default.
+	Format    string `yaml:"format"`
+	Path      string `yaml:"path"`
+	Overwrite bool   `yaml:"overwrite"`
+}
+
+// Select carries the schema/table selection for the split command.
+type Select struct {
+	Schemas        []string `yaml:"schemas"`
+	Tables         []string `yaml:"tables"`
+	ExcludeSchemas []string `yaml:"exclude_schemas"`
+	ExcludeTables  []string `yaml:"exclude_tables"`
+	DatabaseName   string   `yaml:"database_name"`
+}
+
+// LogPolicy is the resolved logfile rotation policy for a job.
+type LogPolicy struct {
+	MaxSizeBytes int64
+	Keep         int
+}
+
+// ResolvedLogPolicy returns the effective rotation policy: the job's own
+// overrides win, then its file's defaults block, then the built-in default.
+func (j *Job) ResolvedLogPolicy() LogPolicy {
+	p := LogPolicy{MaxSizeBytes: defaultLogMaxSizeBytes, Keep: defaultLogKeep}
+
+	if j.fileDefaults != nil {
+		if n, err := parseHumanSize(j.fileDefaults.LogMaxSize); err == nil && n > 0 {
+			p.MaxSizeBytes = n
+		}
+		if j.fileDefaults.LogKeep > 0 {
+			p.Keep = j.fileDefaults.LogKeep
+		}
+	}
+	if n, err := parseHumanSize(j.LogMaxSize); err == nil && n > 0 {
+		p.MaxSizeBytes = n
+	}
+	if j.LogKeep != nil && *j.LogKeep >= 0 {
+		p.Keep = *j.LogKeep
+	}
+	return p
+}
+
+// effectiveDeps returns the union of explicit depends_on entries and the jobs
+// referenced by from_job inputs, deduplicated in stable order.
+func (j *Job) effectiveDeps() []string {
+	seen := map[string]bool{}
+	var deps []string
+	add := func(name string) {
+		if name == "" || name == j.Name || seen[name] {
+			return
+		}
+		seen[name] = true
+		deps = append(deps, name)
+	}
+	for _, d := range j.DependsOn {
+		add(d)
+	}
+	for _, in := range j.Inputs {
+		add(in.FromJob)
+	}
+	return deps
+}
+
+// parseHumanSize parses a byte size such as "5MB", "512 KB", "1gb" or a bare
+// byte count. An empty string returns (0, nil) so callers can fall back.
+func parseHumanSize(s string) (int64, error) {
+	s = strings.TrimSpace(s)
+	if s == "" {
+		return 0, nil
+	}
+	upper := strings.ToUpper(s)
+	mult := int64(1)
+	// Check multi-character suffixes before the bare "B".
+	for _, u := range []struct {
+		suffix string
+		m      int64
+	}{
+		{"KB", 1 << 10}, {"MB", 1 << 20}, {"GB", 1 << 30}, {"B", 1},
+	} {
+		if strings.HasSuffix(upper, u.suffix) {
+			mult = u.m
+			upper = strings.TrimSpace(strings.TrimSuffix(upper, u.suffix))
+			break
+		}
+	}
+	n, err := strconv.ParseFloat(upper, 64)
+	if err != nil {
+		return 0, fmt.Errorf("invalid size %q", s)
+	}
+	if n < 0 {
+		return 0, fmt.Errorf("negative size %q", s)
+	}
+	return int64(n * float64(mult)), nil
+}
+
 // Options carries the subset of command flags a job file may set.
 type Options struct {
 	FlattenSchema   bool   `yaml:"flatten_schema"`
@@ -127,6 +297,9 @@ type Set struct {
 	Files []string
 	// Jobs is keyed by job name.
 	Jobs map[string]*Job
+	// Warnings holds non-fatal load-time messages (e.g. a newer-than-known
+	// schema version). Callers should surface these to the user.
+	Warnings []string
 }

 // Names returns all job names in deterministic (sorted) order.
@@ -199,15 +372,35 @@ func Load(paths []string) (*Set, error) {
 		if err != nil {
 			return nil, fmt.Errorf("failed to read job file %q: %w", path, err)
 		}
+
+		// Peek at the version first so a newer file can be parsed leniently
+		// (unknown fields ignored) instead of failing outright.
+		var probe struct {
+			Version int `yaml:"version"`
+		}
+		if err := yaml.Unmarshal(data, &probe); err != nil {
+			return nil, fmt.Errorf("invalid job file %q: %w", path, err)
+		}
+		version := probe.Version
+		if version == 0 {
+			version = CurrentSchemaVersion
+		}
+		if version < MinSchemaVersion {
+			return nil, fmt.Errorf("job file %q: unsupported version %d (this build accepts %d or newer)", path, version, MinSchemaVersion)
+		}
+		strict := version <= CurrentSchemaVersion
+		if !strict {
+			set.Warnings = append(set.Warnings, fmt.Sprintf(
+				"job file %q declares version %d, newer than this build understands (%d); loading best-effort and ignoring unknown fields",
+				path, version, CurrentSchemaVersion))
+		}
+
 		dec := yaml.NewDecoder(strings.NewReader(string(data)))
-		dec.KnownFields(true)
+		dec.KnownFields(strict)
 		var f File
 		if err := dec.Decode(&f); err != nil {
 			return nil, fmt.Errorf("invalid job file %q: %w", path, err)
 		}
-		if f.Version != SchemaVersion {
-			return nil, fmt.Errorf("job file %q: unsupported version %d (expected %d)", path, f.Version, SchemaVersion)
-		}
 		if len(f.Jobs) == 0 {
 			return nil, fmt.Errorf("job file %q: no jobs defined", path)
 		}
@@ -220,6 +413,7 @@ func Load(paths []string) (*Set, error) {
 			}
 			job.Name = name
 			job.SourceFile = path
+			job.fileDefaults = f.Defaults
 			origin[name] = path
 			set.Jobs[name] = job
 		}
@@ -240,13 +434,30 @@ func (s *Set) Validate() error {
 			errs = append(errs, fmt.Sprintf("job %q: %s", name, msg))
 		}
 	}
-	// Dependency references + cycles.
+	// Dependency references + cycles + from_job wiring.
 	for _, name := range s.Names() {
-		for _, dep := range s.Jobs[name].DependsOn {
+		j := s.Jobs[name]
+		for _, dep := range j.DependsOn {
 			if _, ok := s.Jobs[dep]; !ok {
 				errs = append(errs, fmt.Sprintf("job %q: depends_on unknown job %q", name, dep))
 			}
 		}
+		for i, in := range j.Inputs {
+			if in.FromJob == "" {
+				continue
+			}
+			producer, ok := s.Jobs[in.FromJob]
+			if !ok {
+				errs = append(errs, fmt.Sprintf("job %q: input[%d] from_job references unknown job %q", name, i, in.FromJob))
+				continue
+			}
+			if !producerCommands[producer.Command] || producer.Output == nil ||
+				producer.Output.Path == "" || !SingleFileOutputFormat(producer.Output.Format) {
+				errs = append(errs, fmt.Sprintf(
+					"job %q: input[%d] from_job %q must name a convert/merge/split job that writes a single-file output",
+					name, i, in.FromJob))
+			}
+		}
 	}
 	if cycle := s.findCycle(); cycle != "" {
 		errs = append(errs, fmt.Sprintf("dependency cycle detected: %s", cycle))
@@ -262,7 +473,8 @@ func (j *Job) validate() []string {
 	var e []string

 	switch j.Command {
-	case CommandConvert, CommandMerge, CommandScriptsList, CommandTempl:
+	case CommandConvert, CommandMerge, CommandScriptsList, CommandScriptsExec,
+		CommandTempl, CommandSplit, CommandInspect, CommandDiff:
 	case "":
 		e = append(e, "missing command")
 		return e
@@ -282,6 +494,7 @@ func (j *Job) validate() []string {
 	}
 	checkPath("logfile", j.Logfile)
 	checkPath("template", j.Template)
+	checkPath("rules", j.Rules)
 	for _, in := range j.Inputs {
 		checkPath("input path", in.Path)
 	}
@@ -291,6 +504,13 @@ func (j *Job) validate() []string {
 	if j.Output != nil {
 		checkPath("output path", j.Output.Path)
 	}
+	if j.Report != nil {
+		checkPath("report path", j.Report.Path)
+	}
+
+	if _, err := parseHumanSize(j.LogMaxSize); err != nil {
+		e = append(e, fmt.Sprintf("log_max_size: %v", err))
+	}

 	switch j.Command {
 	case CommandConvert, CommandMerge:
@@ -350,11 +570,113 @@ func (j *Job) validate() []string {
 		if j.Output != nil && j.Output.Format != "" {
 			e = append(e, "output.format is not valid for command \"templ\"")
 		}
+	case CommandSplit:
+		if len(j.Inputs) < 1 {
+			e = append(e, "command \"split\" requires at least 1 input")
+		}
+		for i, in := range j.Inputs {
+			e = append(e, validateInput(i, in)...)
+		}
+		if len(j.ScriptDirs) > 0 {
+			e = append(e, "script_dirs is not valid for command \"split\"")
+		}
+		if j.Report != nil {
+			e = append(e, "report is not valid for command \"split\" (use output)")
+		}
+		if j.Output == nil {
+			e = append(e, "missing output")
+		} else {
+			if j.Output.ConnEnv != "" {
+				e = append(e, "command \"split\" writes a file; output.conn_env is not supported")
+			}
+			e = append(e, validateOutput(*j.Output)...)
+		}
+	case CommandInspect:
+		if len(j.Inputs) < 1 {
+			e = append(e, "command \"inspect\" requires at least 1 input")
+		}
+		for i, in := range j.Inputs {
+			e = append(e, validateInput(i, in)...)
+		}
+		if len(j.ScriptDirs) > 0 {
+			e = append(e, "script_dirs is not valid for command \"inspect\"")
+		}
+		if j.Output != nil {
+			e = append(e, "output is not valid for command \"inspect\" (use report)")
+		}
+		e = append(e, validateReport(j.Report, "inspect", inspectReportFormats, "markdown")...)
+	case CommandDiff:
+		if len(j.Inputs) != 2 {
+			e = append(e, "command \"diff\" requires exactly 2 inputs (source, target)")
+		}
+		for i, in := range j.Inputs {
+			e = append(e, validateInput(i, in)...)
+		}
+		if len(j.ScriptDirs) > 0 {
+			e = append(e, "script_dirs is not valid for command \"diff\"")
+		}
+		if j.Output != nil {
+			e = append(e, "output is not valid for command \"diff\" (use report)")
+		}
+		e = append(e, validateReport(j.Report, "diff", diffReportFormats, "summary")...)
+	case CommandScriptsExec:
+		if len(j.ScriptDirs) == 0 {
+			e = append(e, "command \"scripts-exec\" requires at least one script_dir")
+		}
+		if len(j.Inputs) > 0 {
+			e = append(e, "inputs is not valid for command \"scripts-exec\"")
+		}
+		if j.Report != nil {
+			e = append(e, "report is not valid for command \"scripts-exec\"")
+		}
+		if j.Output == nil || j.Output.ConnEnv == "" {
+			e = append(e, "command \"scripts-exec\" requires output.conn_env (an environment variable name holding a connection string)")
+		} else {
+			if j.Output.Path != "" {
+				e = append(e, "command \"scripts-exec\" executes against a database; output.path is not supported")
+			}
+			f := strings.ToLower(j.Output.Format)
+			if f != "" && f != "pgsql" {
+				e = append(e, fmt.Sprintf("command \"scripts-exec\" only supports pgsql databases (got %q)", j.Output.Format))
+			}
+			if looksLikeSecret(j.Output.ConnEnv) {
+				e = append(e, "output: conn_env must be an environment variable name, not a connection string")
+			}
+		}
+	}
+	return e
+}
+
+// validateReport checks a Report block for the inspect/diff commands.
+func validateReport(r *Report, cmd string, allowed map[string]bool, defFmt string) []string {
+	if r == nil {
+		return []string{fmt.Sprintf("command %q requires a report block", cmd)}
+	}
+	var e []string
+	f := strings.ToLower(r.Format)
+	if f == "" {
+		f = defFmt
+	}
+	if !allowed[f] {
+		names := make([]string, 0, len(allowed))
+		for k := range allowed {
+			names = append(names, k)
+		}
+		sort.Strings(names)
+		e = append(e, fmt.Sprintf("command %q report.format %q is not supported (use: %s)", cmd, r.Format, strings.Join(names, ", ")))
+	}
+	// A diff summary may be written to the log; everything else needs a path.
+	summaryToLog := cmd == "diff" && f == "summary"
+	if r.Path == "" && !summaryToLog {
+		e = append(e, fmt.Sprintf("command %q requires report.path", cmd))
 	}
 	return e
 }

 func validateTemplInput(i int, in Input) []string {
+	if in.FromJob != "" {
+		return fromJobInputShape(i, in)
+	}
 	var e []string
 	if in.Format == "" {
 		return []string{fmt.Sprintf("input[%d]: missing format", i)}
@@ -383,7 +705,27 @@ func validateTemplInput(i int, in Input) []string {
 	return e
 }

+// fromJobInputShape checks the structural rules for an input that pulls its
+// schema from another job's output. The referenced job's existence and kind
+// are checked in Set.Validate, which can see the whole set.
+func fromJobInputShape(i int, in Input) []string {
+	var e []string
+	if in.Path != "" {
+		e = append(e, fmt.Sprintf("input[%d]: from_job takes no path", i))
+	}
+	if in.Format != "" {
+		e = append(e, fmt.Sprintf("input[%d]: from_job inherits the producer's format; drop format", i))
+	}
+	if in.ConnEnv != "" {
+		e = append(e, fmt.Sprintf("input[%d]: from_job takes no conn_env", i))
+	}
+	return e
+}
+
 func validateInput(i int, in Input) []string {
+	if in.FromJob != "" {
+		return fromJobInputShape(i, in)
+	}
 	var e []string
 	if in.Format == "" {
 		e = append(e, fmt.Sprintf("input[%d]: missing format", i))
@@ -487,9 +829,42 @@ func SafeJoin(root, rel string) (string, error) {
 	if rp == ".." || strings.HasPrefix(rp, ".."+string(filepath.Separator)) {
 		return "", fmt.Errorf("path %q escapes the job file directory", rel)
 	}
+	// Symlink hardening: resolve symlinks on the root and on the deepest
+	// existing ancestor of the target, and require the target to still live
+	// inside the resolved root. This catches a symlink inside the job-file
+	// directory that points outside it.
+	realRoot, err := filepath.EvalSymlinks(absRoot)
+	if err != nil {
+		return "", fmt.Errorf("cannot resolve job file directory: %w", err)
+	}
+	realAnc, err := filepath.EvalSymlinks(deepestExistingAncestor(joined))
+	if err != nil {
+		return "", fmt.Errorf("cannot resolve path %q: %w", rel, err)
+	}
+	if realAnc != realRoot {
+		if r, err := filepath.Rel(realRoot, realAnc); err != nil ||
+			r == ".." || strings.HasPrefix(r, ".."+string(filepath.Separator)) {
+			return "", fmt.Errorf("path %q resolves outside the job file directory via a symlink", rel)
+		}
+	}
 	return joined, nil
 }

+// deepestExistingAncestor returns p itself if it exists, otherwise the nearest
+// existing parent directory (falling back to the filesystem root).
+func deepestExistingAncestor(p string) string {
+	for {
+		if _, err := os.Lstat(p); err == nil {
+			return p
+		}
+		parent := filepath.Dir(p)
+		if parent == p {
+			return p
+		}
+		p = parent
+	}
+}
+
 // Plan returns the jobs to execute for name in dependency order. When
 // includeDeps is false only the named job is returned (its declared
 // dependencies are still validated to exist and be acyclic by Validate).
@@ -514,7 +889,7 @@ func (s *Set) Plan(name string, includeDeps bool) ([]*Job, error) {
 		}
 		inProgress[n] = true
 		j := s.Jobs[n]
-		deps := append([]string(nil), j.DependsOn...)
+		deps := j.effectiveDeps()
 		sort.Strings(deps)
 		for _, d := range deps {
 			if _, ok := s.Jobs[d]; !ok {
@@ -543,7 +918,7 @@ func (s *Set) findCycle() string {
 	dfs = func(n string) []string {
 		color[n] = 1
 		stack = append(stack, n)
-		deps := append([]string(nil), s.Jobs[n].DependsOn...)
+		deps := s.Jobs[n].effectiveDeps()
 		sort.Strings(deps)
 		for _, d := range deps {
 			if _, ok := s.Jobs[d]; !ok {