feat(job): complete deferred job-file features
Implements the remaining items from issue #20: - version is now forward-permissive: any value >= 1 is accepted; a newer-than-known version loads best-effort (unknown fields ignored, warning printed) instead of hard-failing on "must be 1" - from_job input reference: `inputs: [{ from_job: <job> }]` resolves to that job's single-file output + format and implies a dependency edge; combined depends_on + from_job graph gets topological ordering and cycle detection - logfile size-rotation, on by default (5MB, keep 3), overridable per job (log_max_size / log_keep) or file-wide via a top-level defaults block - new commands: split (schema/table subsetting via select:), inspect (rule validation -> markdown/json report, fails job on enforced-rule errors), diff (compare exactly two schemas, never fails), scripts-exec (run SQL script dirs against a live PostgreSQL database) - atomic single-file output/report writes (temp file + rename) - symlink-escape hardening in SafeJoin via EvalSymlinks preflight Updates docs/JOB_FILES.md and examples/jobs/relspec.yml accordingly.
This commit is contained in:
Hein
+389
-14
@@ -20,24 +20,52 @@ import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"gopkg.in/yaml.v3"
|
||||
)
|
||||
|
||||
// SchemaVersion is the only job-file schema version this build understands.
|
||||
const SchemaVersion = 1
|
||||
// CurrentSchemaVersion is the highest job-file schema version this build was
|
||||
// written for. MinSchemaVersion is the oldest it still accepts. A file that
|
||||
// declares a version in between loads normally; a newer version loads
|
||||
// best-effort with a warning (see Load); an older-than-minimum version is a
|
||||
// hard error.
|
||||
const (
|
||||
CurrentSchemaVersion = 1
|
||||
MinSchemaVersion = 1
|
||||
)
|
||||
|
||||
// Built-in logfile rotation policy, used when neither the job nor its file's
|
||||
// defaults block sets one.
|
||||
const (
|
||||
defaultLogMaxSizeBytes int64 = 5 << 20 // 5 MiB
|
||||
defaultLogKeep = 3
|
||||
)
|
||||
|
||||
// Command names are a closed allow-list. Arbitrary strings are rejected.
|
||||
const (
|
||||
CommandConvert = "convert" // read one or more schema files, optionally merge, write one output
|
||||
CommandMerge = "merge" // additive merge of two or more schema files into one output
|
||||
CommandScriptsList = "scripts-list" // deterministically list SQL scripts across one or more directories
|
||||
CommandScriptsExec = "scripts-exec" // execute SQL scripts across one or more directories against a live database
|
||||
CommandTempl = "templ" // apply a custom Go text template to one or more schemas
|
||||
CommandSplit = "split" // extract selected schemas/tables into a separate output
|
||||
CommandInspect = "inspect" // validate one or more schemas against rules and write a report
|
||||
CommandDiff = "diff" // compare exactly two schemas and write a differences report
|
||||
)
|
||||
|
||||
// SupportedCommands lists every accepted command, in help order.
|
||||
var SupportedCommands = []string{CommandConvert, CommandMerge, CommandScriptsList, CommandTempl}
|
||||
var SupportedCommands = []string{
|
||||
CommandConvert, CommandMerge, CommandScriptsList, CommandScriptsExec,
|
||||
CommandTempl, CommandSplit, CommandInspect, CommandDiff,
|
||||
}
|
||||
|
||||
// producerCommands are commands whose output is a schema file that another job
|
||||
// may consume via from_job.
|
||||
var producerCommands = map[string]bool{
|
||||
CommandConvert: true, CommandMerge: true, CommandSplit: true,
|
||||
}
|
||||
|
||||
// readerFormats are the file-based input formats a job may declare (path).
|
||||
var readerFormats = map[string]bool{
|
||||
@@ -61,10 +89,41 @@ var writerFormats = map[string]bool{
|
||||
// live database) is supported instead of writing a file.
|
||||
var execOutputFormats = map[string]bool{"pgsql": true}
|
||||
|
||||
// singleFileFormats are output formats that emit exactly one file (as opposed
|
||||
// to a directory of files). Only these are eligible for atomic temp+rename
|
||||
// writes and for being consumed by another job via from_job.
|
||||
var singleFileFormats = map[string]bool{
|
||||
"json": true, "yaml": true, "dbml": true, "dctx": true, "drawdb": true,
|
||||
"graphql": true, "pgsql": true, "mssql": true, "sqlite": true,
|
||||
}
|
||||
|
||||
// SingleFileOutputFormat reports whether format writes exactly one file.
|
||||
func SingleFileOutputFormat(format string) bool {
|
||||
return singleFileFormats[strings.ToLower(format)]
|
||||
}
|
||||
|
||||
// diffReportFormats and inspectReportFormats are the report.format values
|
||||
// accepted by the diff and inspect commands respectively.
|
||||
var (
|
||||
diffReportFormats = map[string]bool{"summary": true, "json": true, "html": true}
|
||||
inspectReportFormats = map[string]bool{"markdown": true, "json": true}
|
||||
)
|
||||
|
||||
// File is the on-disk shape of a single job file.
|
||||
type File struct {
|
||||
Version int `yaml:"version"`
|
||||
Jobs map[string]*Job `yaml:"jobs"`
|
||||
Version int `yaml:"version"`
|
||||
Defaults *Defaults `yaml:"defaults"`
|
||||
Jobs map[string]*Job `yaml:"jobs"`
|
||||
}
|
||||
|
||||
// Defaults carries file-wide settings that individual jobs may override.
|
||||
type Defaults struct {
|
||||
// LogMaxSize is a human-readable size ("5MB", "512KB", "1GB"). Empty
|
||||
// means "use the built-in default".
|
||||
LogMaxSize string `yaml:"log_max_size"`
|
||||
// LogKeep is how many rotated logfiles to retain. Zero means "use the
|
||||
// built-in default".
|
||||
LogKeep int `yaml:"log_keep"`
|
||||
}
|
||||
|
||||
// Job is one named job within a job file.
|
||||
@@ -72,6 +131,9 @@ type Job struct {
|
||||
// Name and SourceFile are populated by Load, not parsed from YAML.
|
||||
Name string `yaml:"-"`
|
||||
SourceFile string `yaml:"-"`
|
||||
// fileDefaults is the Defaults block of the file that declared this job,
|
||||
// captured by Load. nil when the file had none.
|
||||
fileDefaults *Defaults `yaml:"-"`
|
||||
|
||||
Command string `yaml:"command"`
|
||||
Description string `yaml:"description"`
|
||||
@@ -82,8 +144,13 @@ type Job struct {
|
||||
Mode string `yaml:"mode"`
|
||||
FilenamePattern string `yaml:"filename_pattern"`
|
||||
Output *Output `yaml:"output"`
|
||||
Rules string `yaml:"rules"`
|
||||
Report *Report `yaml:"report"`
|
||||
Select *Select `yaml:"select"`
|
||||
Options Options `yaml:"options"`
|
||||
Logfile string `yaml:"logfile"`
|
||||
LogMaxSize string `yaml:"log_max_size"`
|
||||
LogKeep *int `yaml:"log_keep"`
|
||||
}
|
||||
|
||||
// Input is one declared input schema.
|
||||
@@ -94,6 +161,10 @@ type Input struct {
|
||||
// ConnEnv is the NAME of an environment variable holding a connection
|
||||
// string, used with database formats. The value is never stored here.
|
||||
ConnEnv string `yaml:"conn_env"`
|
||||
// FromJob names another job in the set whose file output is used as this
|
||||
// input. It implies a dependency on that job. Path/Format/ConnEnv must be
|
||||
// empty when FromJob is set; the format is inherited from the producer.
|
||||
FromJob string `yaml:"from_job"`
|
||||
}
|
||||
|
||||
// Output is the declared output target.
|
||||
@@ -104,6 +175,105 @@ type Output struct {
|
||||
Overwrite bool `yaml:"overwrite"`
|
||||
}
|
||||
|
||||
// Report is the output target for the inspect and diff commands.
|
||||
type Report struct {
|
||||
// Format is the report format: diff accepts summary|json|html, inspect
|
||||
// accepts markdown|json. Empty means the command's default.
|
||||
Format string `yaml:"format"`
|
||||
Path string `yaml:"path"`
|
||||
Overwrite bool `yaml:"overwrite"`
|
||||
}
|
||||
|
||||
// Select carries the schema/table selection for the split command.
|
||||
type Select struct {
|
||||
Schemas []string `yaml:"schemas"`
|
||||
Tables []string `yaml:"tables"`
|
||||
ExcludeSchemas []string `yaml:"exclude_schemas"`
|
||||
ExcludeTables []string `yaml:"exclude_tables"`
|
||||
DatabaseName string `yaml:"database_name"`
|
||||
}
|
||||
|
||||
// LogPolicy is the resolved logfile rotation policy for a job.
|
||||
type LogPolicy struct {
|
||||
MaxSizeBytes int64
|
||||
Keep int
|
||||
}
|
||||
|
||||
// ResolvedLogPolicy returns the effective rotation policy: the job's own
|
||||
// overrides win, then its file's defaults block, then the built-in default.
|
||||
func (j *Job) ResolvedLogPolicy() LogPolicy {
|
||||
p := LogPolicy{MaxSizeBytes: defaultLogMaxSizeBytes, Keep: defaultLogKeep}
|
||||
|
||||
if j.fileDefaults != nil {
|
||||
if n, err := parseHumanSize(j.fileDefaults.LogMaxSize); err == nil && n > 0 {
|
||||
p.MaxSizeBytes = n
|
||||
}
|
||||
if j.fileDefaults.LogKeep > 0 {
|
||||
p.Keep = j.fileDefaults.LogKeep
|
||||
}
|
||||
}
|
||||
if n, err := parseHumanSize(j.LogMaxSize); err == nil && n > 0 {
|
||||
p.MaxSizeBytes = n
|
||||
}
|
||||
if j.LogKeep != nil && *j.LogKeep >= 0 {
|
||||
p.Keep = *j.LogKeep
|
||||
}
|
||||
return p
|
||||
}
|
||||
|
||||
// effectiveDeps returns the union of explicit depends_on entries and the jobs
|
||||
// referenced by from_job inputs, deduplicated in stable order.
|
||||
func (j *Job) effectiveDeps() []string {
|
||||
seen := map[string]bool{}
|
||||
var deps []string
|
||||
add := func(name string) {
|
||||
if name == "" || name == j.Name || seen[name] {
|
||||
return
|
||||
}
|
||||
seen[name] = true
|
||||
deps = append(deps, name)
|
||||
}
|
||||
for _, d := range j.DependsOn {
|
||||
add(d)
|
||||
}
|
||||
for _, in := range j.Inputs {
|
||||
add(in.FromJob)
|
||||
}
|
||||
return deps
|
||||
}
|
||||
|
||||
// parseHumanSize parses a byte size such as "5MB", "512 KB", "1gb" or a bare
|
||||
// byte count. An empty string returns (0, nil) so callers can fall back.
|
||||
func parseHumanSize(s string) (int64, error) {
|
||||
s = strings.TrimSpace(s)
|
||||
if s == "" {
|
||||
return 0, nil
|
||||
}
|
||||
upper := strings.ToUpper(s)
|
||||
mult := int64(1)
|
||||
// Check multi-character suffixes before the bare "B".
|
||||
for _, u := range []struct {
|
||||
suffix string
|
||||
m int64
|
||||
}{
|
||||
{"KB", 1 << 10}, {"MB", 1 << 20}, {"GB", 1 << 30}, {"B", 1},
|
||||
} {
|
||||
if strings.HasSuffix(upper, u.suffix) {
|
||||
mult = u.m
|
||||
upper = strings.TrimSpace(strings.TrimSuffix(upper, u.suffix))
|
||||
break
|
||||
}
|
||||
}
|
||||
n, err := strconv.ParseFloat(upper, 64)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("invalid size %q", s)
|
||||
}
|
||||
if n < 0 {
|
||||
return 0, fmt.Errorf("negative size %q", s)
|
||||
}
|
||||
return int64(n * float64(mult)), nil
|
||||
}
|
||||
|
||||
// Options carries the subset of command flags a job file may set.
|
||||
type Options struct {
|
||||
FlattenSchema bool `yaml:"flatten_schema"`
|
||||
@@ -127,6 +297,9 @@ type Set struct {
|
||||
Files []string
|
||||
// Jobs is keyed by job name.
|
||||
Jobs map[string]*Job
|
||||
// Warnings holds non-fatal load-time messages (e.g. a newer-than-known
|
||||
// schema version). Callers should surface these to the user.
|
||||
Warnings []string
|
||||
}
|
||||
|
||||
// Names returns all job names in deterministic (sorted) order.
|
||||
@@ -199,15 +372,35 @@ func Load(paths []string) (*Set, error) {
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to read job file %q: %w", path, err)
|
||||
}
|
||||
|
||||
// Peek at the version first so a newer file can be parsed leniently
|
||||
// (unknown fields ignored) instead of failing outright.
|
||||
var probe struct {
|
||||
Version int `yaml:"version"`
|
||||
}
|
||||
if err := yaml.Unmarshal(data, &probe); err != nil {
|
||||
return nil, fmt.Errorf("invalid job file %q: %w", path, err)
|
||||
}
|
||||
version := probe.Version
|
||||
if version == 0 {
|
||||
version = CurrentSchemaVersion
|
||||
}
|
||||
if version < MinSchemaVersion {
|
||||
return nil, fmt.Errorf("job file %q: unsupported version %d (this build accepts %d or newer)", path, version, MinSchemaVersion)
|
||||
}
|
||||
strict := version <= CurrentSchemaVersion
|
||||
if !strict {
|
||||
set.Warnings = append(set.Warnings, fmt.Sprintf(
|
||||
"job file %q declares version %d, newer than this build understands (%d); loading best-effort and ignoring unknown fields",
|
||||
path, version, CurrentSchemaVersion))
|
||||
}
|
||||
|
||||
dec := yaml.NewDecoder(strings.NewReader(string(data)))
|
||||
dec.KnownFields(true)
|
||||
dec.KnownFields(strict)
|
||||
var f File
|
||||
if err := dec.Decode(&f); err != nil {
|
||||
return nil, fmt.Errorf("invalid job file %q: %w", path, err)
|
||||
}
|
||||
if f.Version != SchemaVersion {
|
||||
return nil, fmt.Errorf("job file %q: unsupported version %d (expected %d)", path, f.Version, SchemaVersion)
|
||||
}
|
||||
if len(f.Jobs) == 0 {
|
||||
return nil, fmt.Errorf("job file %q: no jobs defined", path)
|
||||
}
|
||||
@@ -220,6 +413,7 @@ func Load(paths []string) (*Set, error) {
|
||||
}
|
||||
job.Name = name
|
||||
job.SourceFile = path
|
||||
job.fileDefaults = f.Defaults
|
||||
origin[name] = path
|
||||
set.Jobs[name] = job
|
||||
}
|
||||
@@ -240,13 +434,30 @@ func (s *Set) Validate() error {
|
||||
errs = append(errs, fmt.Sprintf("job %q: %s", name, msg))
|
||||
}
|
||||
}
|
||||
// Dependency references + cycles.
|
||||
// Dependency references + cycles + from_job wiring.
|
||||
for _, name := range s.Names() {
|
||||
for _, dep := range s.Jobs[name].DependsOn {
|
||||
j := s.Jobs[name]
|
||||
for _, dep := range j.DependsOn {
|
||||
if _, ok := s.Jobs[dep]; !ok {
|
||||
errs = append(errs, fmt.Sprintf("job %q: depends_on unknown job %q", name, dep))
|
||||
}
|
||||
}
|
||||
for i, in := range j.Inputs {
|
||||
if in.FromJob == "" {
|
||||
continue
|
||||
}
|
||||
producer, ok := s.Jobs[in.FromJob]
|
||||
if !ok {
|
||||
errs = append(errs, fmt.Sprintf("job %q: input[%d] from_job references unknown job %q", name, i, in.FromJob))
|
||||
continue
|
||||
}
|
||||
if !producerCommands[producer.Command] || producer.Output == nil ||
|
||||
producer.Output.Path == "" || !SingleFileOutputFormat(producer.Output.Format) {
|
||||
errs = append(errs, fmt.Sprintf(
|
||||
"job %q: input[%d] from_job %q must name a convert/merge/split job that writes a single-file output",
|
||||
name, i, in.FromJob))
|
||||
}
|
||||
}
|
||||
}
|
||||
if cycle := s.findCycle(); cycle != "" {
|
||||
errs = append(errs, fmt.Sprintf("dependency cycle detected: %s", cycle))
|
||||
@@ -262,7 +473,8 @@ func (j *Job) validate() []string {
|
||||
var e []string
|
||||
|
||||
switch j.Command {
|
||||
case CommandConvert, CommandMerge, CommandScriptsList, CommandTempl:
|
||||
case CommandConvert, CommandMerge, CommandScriptsList, CommandScriptsExec,
|
||||
CommandTempl, CommandSplit, CommandInspect, CommandDiff:
|
||||
case "":
|
||||
e = append(e, "missing command")
|
||||
return e
|
||||
@@ -282,6 +494,7 @@ func (j *Job) validate() []string {
|
||||
}
|
||||
checkPath("logfile", j.Logfile)
|
||||
checkPath("template", j.Template)
|
||||
checkPath("rules", j.Rules)
|
||||
for _, in := range j.Inputs {
|
||||
checkPath("input path", in.Path)
|
||||
}
|
||||
@@ -291,6 +504,13 @@ func (j *Job) validate() []string {
|
||||
if j.Output != nil {
|
||||
checkPath("output path", j.Output.Path)
|
||||
}
|
||||
if j.Report != nil {
|
||||
checkPath("report path", j.Report.Path)
|
||||
}
|
||||
|
||||
if _, err := parseHumanSize(j.LogMaxSize); err != nil {
|
||||
e = append(e, fmt.Sprintf("log_max_size: %v", err))
|
||||
}
|
||||
|
||||
switch j.Command {
|
||||
case CommandConvert, CommandMerge:
|
||||
@@ -350,11 +570,113 @@ func (j *Job) validate() []string {
|
||||
if j.Output != nil && j.Output.Format != "" {
|
||||
e = append(e, "output.format is not valid for command \"templ\"")
|
||||
}
|
||||
case CommandSplit:
|
||||
if len(j.Inputs) < 1 {
|
||||
e = append(e, "command \"split\" requires at least 1 input")
|
||||
}
|
||||
for i, in := range j.Inputs {
|
||||
e = append(e, validateInput(i, in)...)
|
||||
}
|
||||
if len(j.ScriptDirs) > 0 {
|
||||
e = append(e, "script_dirs is not valid for command \"split\"")
|
||||
}
|
||||
if j.Report != nil {
|
||||
e = append(e, "report is not valid for command \"split\" (use output)")
|
||||
}
|
||||
if j.Output == nil {
|
||||
e = append(e, "missing output")
|
||||
} else {
|
||||
if j.Output.ConnEnv != "" {
|
||||
e = append(e, "command \"split\" writes a file; output.conn_env is not supported")
|
||||
}
|
||||
e = append(e, validateOutput(*j.Output)...)
|
||||
}
|
||||
case CommandInspect:
|
||||
if len(j.Inputs) < 1 {
|
||||
e = append(e, "command \"inspect\" requires at least 1 input")
|
||||
}
|
||||
for i, in := range j.Inputs {
|
||||
e = append(e, validateInput(i, in)...)
|
||||
}
|
||||
if len(j.ScriptDirs) > 0 {
|
||||
e = append(e, "script_dirs is not valid for command \"inspect\"")
|
||||
}
|
||||
if j.Output != nil {
|
||||
e = append(e, "output is not valid for command \"inspect\" (use report)")
|
||||
}
|
||||
e = append(e, validateReport(j.Report, "inspect", inspectReportFormats, "markdown")...)
|
||||
case CommandDiff:
|
||||
if len(j.Inputs) != 2 {
|
||||
e = append(e, "command \"diff\" requires exactly 2 inputs (source, target)")
|
||||
}
|
||||
for i, in := range j.Inputs {
|
||||
e = append(e, validateInput(i, in)...)
|
||||
}
|
||||
if len(j.ScriptDirs) > 0 {
|
||||
e = append(e, "script_dirs is not valid for command \"diff\"")
|
||||
}
|
||||
if j.Output != nil {
|
||||
e = append(e, "output is not valid for command \"diff\" (use report)")
|
||||
}
|
||||
e = append(e, validateReport(j.Report, "diff", diffReportFormats, "summary")...)
|
||||
case CommandScriptsExec:
|
||||
if len(j.ScriptDirs) == 0 {
|
||||
e = append(e, "command \"scripts-exec\" requires at least one script_dir")
|
||||
}
|
||||
if len(j.Inputs) > 0 {
|
||||
e = append(e, "inputs is not valid for command \"scripts-exec\"")
|
||||
}
|
||||
if j.Report != nil {
|
||||
e = append(e, "report is not valid for command \"scripts-exec\"")
|
||||
}
|
||||
if j.Output == nil || j.Output.ConnEnv == "" {
|
||||
e = append(e, "command \"scripts-exec\" requires output.conn_env (an environment variable name holding a connection string)")
|
||||
} else {
|
||||
if j.Output.Path != "" {
|
||||
e = append(e, "command \"scripts-exec\" executes against a database; output.path is not supported")
|
||||
}
|
||||
f := strings.ToLower(j.Output.Format)
|
||||
if f != "" && f != "pgsql" {
|
||||
e = append(e, fmt.Sprintf("command \"scripts-exec\" only supports pgsql databases (got %q)", j.Output.Format))
|
||||
}
|
||||
if looksLikeSecret(j.Output.ConnEnv) {
|
||||
e = append(e, "output: conn_env must be an environment variable name, not a connection string")
|
||||
}
|
||||
}
|
||||
}
|
||||
return e
|
||||
}
|
||||
|
||||
// validateReport checks a Report block for the inspect/diff commands.
|
||||
func validateReport(r *Report, cmd string, allowed map[string]bool, defFmt string) []string {
|
||||
if r == nil {
|
||||
return []string{fmt.Sprintf("command %q requires a report block", cmd)}
|
||||
}
|
||||
var e []string
|
||||
f := strings.ToLower(r.Format)
|
||||
if f == "" {
|
||||
f = defFmt
|
||||
}
|
||||
if !allowed[f] {
|
||||
names := make([]string, 0, len(allowed))
|
||||
for k := range allowed {
|
||||
names = append(names, k)
|
||||
}
|
||||
sort.Strings(names)
|
||||
e = append(e, fmt.Sprintf("command %q report.format %q is not supported (use: %s)", cmd, r.Format, strings.Join(names, ", ")))
|
||||
}
|
||||
// A diff summary may be written to the log; everything else needs a path.
|
||||
summaryToLog := cmd == "diff" && f == "summary"
|
||||
if r.Path == "" && !summaryToLog {
|
||||
e = append(e, fmt.Sprintf("command %q requires report.path", cmd))
|
||||
}
|
||||
return e
|
||||
}
|
||||
|
||||
func validateTemplInput(i int, in Input) []string {
|
||||
if in.FromJob != "" {
|
||||
return fromJobInputShape(i, in)
|
||||
}
|
||||
var e []string
|
||||
if in.Format == "" {
|
||||
return []string{fmt.Sprintf("input[%d]: missing format", i)}
|
||||
@@ -383,7 +705,27 @@ func validateTemplInput(i int, in Input) []string {
|
||||
return e
|
||||
}
|
||||
|
||||
// fromJobInputShape checks the structural rules for an input that pulls its
|
||||
// schema from another job's output. The referenced job's existence and kind
|
||||
// are checked in Set.Validate, which can see the whole set.
|
||||
func fromJobInputShape(i int, in Input) []string {
|
||||
var e []string
|
||||
if in.Path != "" {
|
||||
e = append(e, fmt.Sprintf("input[%d]: from_job takes no path", i))
|
||||
}
|
||||
if in.Format != "" {
|
||||
e = append(e, fmt.Sprintf("input[%d]: from_job inherits the producer's format; drop format", i))
|
||||
}
|
||||
if in.ConnEnv != "" {
|
||||
e = append(e, fmt.Sprintf("input[%d]: from_job takes no conn_env", i))
|
||||
}
|
||||
return e
|
||||
}
|
||||
|
||||
func validateInput(i int, in Input) []string {
|
||||
if in.FromJob != "" {
|
||||
return fromJobInputShape(i, in)
|
||||
}
|
||||
var e []string
|
||||
if in.Format == "" {
|
||||
e = append(e, fmt.Sprintf("input[%d]: missing format", i))
|
||||
@@ -487,9 +829,42 @@ func SafeJoin(root, rel string) (string, error) {
|
||||
if rp == ".." || strings.HasPrefix(rp, ".."+string(filepath.Separator)) {
|
||||
return "", fmt.Errorf("path %q escapes the job file directory", rel)
|
||||
}
|
||||
// Symlink hardening: resolve symlinks on the root and on the deepest
|
||||
// existing ancestor of the target, and require the target to still live
|
||||
// inside the resolved root. This catches a symlink inside the job-file
|
||||
// directory that points outside it.
|
||||
realRoot, err := filepath.EvalSymlinks(absRoot)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("cannot resolve job file directory: %w", err)
|
||||
}
|
||||
realAnc, err := filepath.EvalSymlinks(deepestExistingAncestor(joined))
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("cannot resolve path %q: %w", rel, err)
|
||||
}
|
||||
if realAnc != realRoot {
|
||||
if r, err := filepath.Rel(realRoot, realAnc); err != nil ||
|
||||
r == ".." || strings.HasPrefix(r, ".."+string(filepath.Separator)) {
|
||||
return "", fmt.Errorf("path %q resolves outside the job file directory via a symlink", rel)
|
||||
}
|
||||
}
|
||||
return joined, nil
|
||||
}
|
||||
|
||||
// deepestExistingAncestor returns p itself if it exists, otherwise the nearest
|
||||
// existing parent directory (falling back to the filesystem root).
|
||||
func deepestExistingAncestor(p string) string {
|
||||
for {
|
||||
if _, err := os.Lstat(p); err == nil {
|
||||
return p
|
||||
}
|
||||
parent := filepath.Dir(p)
|
||||
if parent == p {
|
||||
return p
|
||||
}
|
||||
p = parent
|
||||
}
|
||||
}
|
||||
|
||||
// Plan returns the jobs to execute for name in dependency order. When
|
||||
// includeDeps is false only the named job is returned (its declared
|
||||
// dependencies are still validated to exist and be acyclic by Validate).
|
||||
@@ -514,7 +889,7 @@ func (s *Set) Plan(name string, includeDeps bool) ([]*Job, error) {
|
||||
}
|
||||
inProgress[n] = true
|
||||
j := s.Jobs[n]
|
||||
deps := append([]string(nil), j.DependsOn...)
|
||||
deps := j.effectiveDeps()
|
||||
sort.Strings(deps)
|
||||
for _, d := range deps {
|
||||
if _, ok := s.Jobs[d]; !ok {
|
||||
@@ -543,7 +918,7 @@ func (s *Set) findCycle() string {
|
||||
dfs = func(n string) []string {
|
||||
color[n] = 1
|
||||
stack = append(stack, n)
|
||||
deps := append([]string(nil), s.Jobs[n].DependsOn...)
|
||||
deps := s.Jobs[n].effectiveDeps()
|
||||
sort.Strings(deps)
|
||||
for _, d := range deps {
|
||||
if _, ok := s.Jobs[d]; !ok {
|
||||
|
||||
+203
-5
@@ -56,13 +56,82 @@ func TestLoadRejectsUnknownFields(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadRejectsBadVersion(t *testing.T) {
|
||||
func TestLoadWarnsOnNewerVersion(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
p := filepath.Join(dir, "relspec.yml")
|
||||
write(t, p, "version: 2\njobs:\n a:\n command: convert\n")
|
||||
_, err := Load([]string{p})
|
||||
if err == nil || !strings.Contains(err.Error(), "unsupported version") {
|
||||
t.Fatalf("expected unsupported version error, got %v", err)
|
||||
// A newer version loads best-effort with a warning, and unknown fields
|
||||
// from the newer schema are ignored rather than rejected.
|
||||
write(t, p, "version: 99\njobs:\n a:\n command: convert\n"+
|
||||
" inputs:\n - path: a.dbml\n format: dbml\n"+
|
||||
" output:\n format: json\n path: out.json\n"+
|
||||
" future_field: whatever\n")
|
||||
set, err := Load([]string{p})
|
||||
if err != nil {
|
||||
t.Fatalf("newer version should load, got %v", err)
|
||||
}
|
||||
if len(set.Warnings) == 0 {
|
||||
t.Fatal("expected a warning about the newer version")
|
||||
}
|
||||
if err := set.Validate(); err != nil {
|
||||
t.Fatalf("validate: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadAcceptsOmittedVersion(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
p := filepath.Join(dir, "relspec.yml")
|
||||
write(t, p, "jobs:\n a:\n command: convert\n"+
|
||||
" inputs:\n - path: a.dbml\n format: dbml\n"+
|
||||
" output:\n format: json\n path: out.json\n")
|
||||
set, err := Load([]string{p})
|
||||
if err != nil {
|
||||
t.Fatalf("omitted version should load, got %v", err)
|
||||
}
|
||||
if len(set.Warnings) != 0 {
|
||||
t.Fatalf("omitted version should not warn, got %v", set.Warnings)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadStillRejectsUnknownFieldsAtCurrentVersion(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
p := filepath.Join(dir, "relspec.yml")
|
||||
write(t, p, "version: 1\njobs:\n a:\n command: convert\n bogus: true\n")
|
||||
if _, err := Load([]string{p}); err == nil {
|
||||
t.Fatal("expected unknown-field rejection at the current version")
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseHumanSize(t *testing.T) {
|
||||
cases := []struct {
|
||||
in string
|
||||
want int64
|
||||
bad bool
|
||||
}{
|
||||
{"", 0, false},
|
||||
{"512", 512, false},
|
||||
{"512B", 512, false},
|
||||
{"1KB", 1 << 10, false},
|
||||
{"5MB", 5 << 20, false},
|
||||
{"1gb", 1 << 30, false},
|
||||
{" 2 MB ", 2 << 20, false},
|
||||
{"nonsense", 0, true},
|
||||
{"-1MB", 0, true},
|
||||
}
|
||||
for _, c := range cases {
|
||||
got, err := parseHumanSize(c.in)
|
||||
if c.bad {
|
||||
if err == nil {
|
||||
t.Errorf("parseHumanSize(%q): expected error", c.in)
|
||||
}
|
||||
continue
|
||||
}
|
||||
if err != nil {
|
||||
t.Errorf("parseHumanSize(%q): %v", c.in, err)
|
||||
continue
|
||||
}
|
||||
if got != c.want {
|
||||
t.Errorf("parseHumanSize(%q) = %d, want %d", c.in, got, c.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -239,6 +308,135 @@ func TestShippedExampleIsValid(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestFromJobWiring(t *testing.T) {
|
||||
content := "version: 1\njobs:\n" +
|
||||
" producer:\n command: convert\n" +
|
||||
" inputs:\n - path: a.dbml\n format: dbml\n" +
|
||||
" output:\n format: json\n path: build/schema.json\n" +
|
||||
" consumer:\n command: convert\n" +
|
||||
" inputs:\n - from_job: producer\n" +
|
||||
" output:\n format: yaml\n path: build/schema.yaml\n"
|
||||
set := loadOne(t, content)
|
||||
if err := set.Validate(); err != nil {
|
||||
t.Fatalf("validate: %v", err)
|
||||
}
|
||||
plan, err := set.Plan("consumer", true)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(plan) != 2 || plan[0].Name != "producer" || plan[1].Name != "consumer" {
|
||||
t.Fatalf("plan = %v, want [producer consumer]", plan)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFromJobRejectsNonProducer(t *testing.T) {
|
||||
content := "version: 1\njobs:\n" +
|
||||
" lister:\n command: scripts-list\n script_dirs: [migrations]\n" +
|
||||
" consumer:\n command: convert\n" +
|
||||
" inputs:\n - from_job: lister\n" +
|
||||
" output:\n format: yaml\n path: out.yaml\n"
|
||||
set := loadOne(t, content)
|
||||
if err := set.Validate(); err == nil || !strings.Contains(err.Error(), "from_job") {
|
||||
t.Fatalf("want from_job producer error, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFromJobRejectsUnknownJob(t *testing.T) {
|
||||
content := "version: 1\njobs:\n" +
|
||||
" consumer:\n command: convert\n" +
|
||||
" inputs:\n - from_job: ghost\n" +
|
||||
" output:\n format: yaml\n path: out.yaml\n"
|
||||
set := loadOne(t, content)
|
||||
if err := set.Validate(); err == nil || !strings.Contains(err.Error(), "unknown job") {
|
||||
t.Fatalf("want unknown job error, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFromJobCycleDetected(t *testing.T) {
|
||||
content := "version: 1\njobs:\n" +
|
||||
" a:\n command: convert\n" +
|
||||
" inputs:\n - from_job: b\n" +
|
||||
" output:\n format: json\n path: a.json\n" +
|
||||
" b:\n command: convert\n" +
|
||||
" inputs:\n - from_job: a\n" +
|
||||
" output:\n format: json\n path: b.json\n"
|
||||
set := loadOne(t, content)
|
||||
if err := set.Validate(); err == nil || !strings.Contains(err.Error(), "cycle") {
|
||||
t.Fatalf("want cycle error, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSplitJobValidation(t *testing.T) {
|
||||
set := loadOne(t, "version: 1\njobs:\n s:\n command: split\n"+
|
||||
" inputs:\n - path: a.dbml\n format: dbml\n")
|
||||
if err := set.Validate(); err == nil || !strings.Contains(err.Error(), "missing output") {
|
||||
t.Fatalf("want missing output, got %v", err)
|
||||
}
|
||||
set = loadOne(t, "version: 1\njobs:\n s:\n command: split\n"+
|
||||
" inputs:\n - path: a.dbml\n format: dbml\n"+
|
||||
" select:\n tables: [users]\n"+
|
||||
" output:\n format: json\n path: out.json\n")
|
||||
if err := set.Validate(); err != nil {
|
||||
t.Fatalf("expected valid split job, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestInspectJobValidation(t *testing.T) {
|
||||
set := loadOne(t, "version: 1\njobs:\n i:\n command: inspect\n"+
|
||||
" inputs:\n - path: a.dbml\n format: dbml\n")
|
||||
if err := set.Validate(); err == nil || !strings.Contains(err.Error(), "report") {
|
||||
t.Fatalf("want report required, got %v", err)
|
||||
}
|
||||
set = loadOne(t, "version: 1\njobs:\n i:\n command: inspect\n"+
|
||||
" inputs:\n - path: a.dbml\n format: dbml\n"+
|
||||
" report:\n format: json\n path: build/report.json\n")
|
||||
if err := set.Validate(); err != nil {
|
||||
t.Fatalf("expected valid inspect job, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDiffJobValidation(t *testing.T) {
|
||||
set := loadOne(t, "version: 1\njobs:\n d:\n command: diff\n"+
|
||||
" inputs:\n - path: a.dbml\n format: dbml\n"+
|
||||
" report:\n format: summary\n")
|
||||
if err := set.Validate(); err == nil || !strings.Contains(err.Error(), "exactly 2 inputs") {
|
||||
t.Fatalf("want exactly 2 inputs, got %v", err)
|
||||
}
|
||||
set = loadOne(t, "version: 1\njobs:\n d:\n command: diff\n"+
|
||||
" inputs:\n - path: a.dbml\n format: dbml\n"+
|
||||
" - path: b.dbml\n format: dbml\n"+
|
||||
" report:\n format: summary\n")
|
||||
if err := set.Validate(); err != nil {
|
||||
t.Fatalf("expected valid diff job, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestScriptsExecValidation(t *testing.T) {
|
||||
set := loadOne(t, "version: 1\njobs:\n x:\n command: scripts-exec\n"+
|
||||
" script_dirs: [migrations]\n")
|
||||
if err := set.Validate(); err == nil || !strings.Contains(err.Error(), "conn_env") {
|
||||
t.Fatalf("want output.conn_env required, got %v", err)
|
||||
}
|
||||
set = loadOne(t, "version: 1\njobs:\n x:\n command: scripts-exec\n"+
|
||||
" script_dirs: [migrations]\n"+
|
||||
" output:\n conn_env: TARGET_DB_URL\n")
|
||||
if err := set.Validate(); err != nil {
|
||||
t.Fatalf("expected valid scripts-exec job, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSafeJoinRejectsSymlinkEscape(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
outside := t.TempDir()
|
||||
link := filepath.Join(root, "link")
|
||||
if err := os.Symlink(outside, link); err != nil {
|
||||
t.Skipf("symlink not supported: %v", err)
|
||||
}
|
||||
if _, err := SafeJoin(root, "link/x.sql"); err == nil {
|
||||
t.Fatal("expected rejection of a path escaping via a symlink")
|
||||
}
|
||||
}
|
||||
|
||||
func TestScriptsListValidation(t *testing.T) {
|
||||
set := loadOne(t, "version: 1\njobs:\n s:\n command: scripts-list\n")
|
||||
if err := set.Validate(); err == nil || !strings.Contains(err.Error(), "script_dir") {
|
||||
|
||||
Reference in New Issue
Block a user