feat(job): complete deferred job-file features
Implements the remaining items from issue #20: - version is now forward-permissive: any value >= 1 is accepted; a newer-than-known version loads best-effort (unknown fields ignored, warning printed) instead of hard-failing on "must be 1" - from_job input reference: `inputs: [{ from_job: <job> }]` resolves to that job's single-file output + format and implies a dependency edge; combined depends_on + from_job graph gets topological ordering and cycle detection - logfile size-rotation, on by default (5MB, keep 3), overridable per job (log_max_size / log_keep) or file-wide via a top-level defaults block - new commands: split (schema/table subsetting via select:), inspect (rule validation -> markdown/json report, fails job on enforced-rule errors), diff (compare exactly two schemas, never fails), scripts-exec (run SQL script dirs against a live PostgreSQL database) - atomic single-file output/report writes (temp file + rename) - symlink-escape hardening in SafeJoin via EvalSymlinks preflight Updates docs/JOB_FILES.md and examples/jobs/relspec.yml accordingly.
This commit is contained in:
Hein
+203
-5
@@ -56,13 +56,82 @@ func TestLoadRejectsUnknownFields(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadRejectsBadVersion(t *testing.T) {
|
||||
func TestLoadWarnsOnNewerVersion(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
p := filepath.Join(dir, "relspec.yml")
|
||||
write(t, p, "version: 2\njobs:\n a:\n command: convert\n")
|
||||
_, err := Load([]string{p})
|
||||
if err == nil || !strings.Contains(err.Error(), "unsupported version") {
|
||||
t.Fatalf("expected unsupported version error, got %v", err)
|
||||
// A newer version loads best-effort with a warning, and unknown fields
|
||||
// from the newer schema are ignored rather than rejected.
|
||||
write(t, p, "version: 99\njobs:\n a:\n command: convert\n"+
|
||||
" inputs:\n - path: a.dbml\n format: dbml\n"+
|
||||
" output:\n format: json\n path: out.json\n"+
|
||||
" future_field: whatever\n")
|
||||
set, err := Load([]string{p})
|
||||
if err != nil {
|
||||
t.Fatalf("newer version should load, got %v", err)
|
||||
}
|
||||
if len(set.Warnings) == 0 {
|
||||
t.Fatal("expected a warning about the newer version")
|
||||
}
|
||||
if err := set.Validate(); err != nil {
|
||||
t.Fatalf("validate: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadAcceptsOmittedVersion(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
p := filepath.Join(dir, "relspec.yml")
|
||||
write(t, p, "jobs:\n a:\n command: convert\n"+
|
||||
" inputs:\n - path: a.dbml\n format: dbml\n"+
|
||||
" output:\n format: json\n path: out.json\n")
|
||||
set, err := Load([]string{p})
|
||||
if err != nil {
|
||||
t.Fatalf("omitted version should load, got %v", err)
|
||||
}
|
||||
if len(set.Warnings) != 0 {
|
||||
t.Fatalf("omitted version should not warn, got %v", set.Warnings)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadStillRejectsUnknownFieldsAtCurrentVersion(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
p := filepath.Join(dir, "relspec.yml")
|
||||
write(t, p, "version: 1\njobs:\n a:\n command: convert\n bogus: true\n")
|
||||
if _, err := Load([]string{p}); err == nil {
|
||||
t.Fatal("expected unknown-field rejection at the current version")
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseHumanSize(t *testing.T) {
|
||||
cases := []struct {
|
||||
in string
|
||||
want int64
|
||||
bad bool
|
||||
}{
|
||||
{"", 0, false},
|
||||
{"512", 512, false},
|
||||
{"512B", 512, false},
|
||||
{"1KB", 1 << 10, false},
|
||||
{"5MB", 5 << 20, false},
|
||||
{"1gb", 1 << 30, false},
|
||||
{" 2 MB ", 2 << 20, false},
|
||||
{"nonsense", 0, true},
|
||||
{"-1MB", 0, true},
|
||||
}
|
||||
for _, c := range cases {
|
||||
got, err := parseHumanSize(c.in)
|
||||
if c.bad {
|
||||
if err == nil {
|
||||
t.Errorf("parseHumanSize(%q): expected error", c.in)
|
||||
}
|
||||
continue
|
||||
}
|
||||
if err != nil {
|
||||
t.Errorf("parseHumanSize(%q): %v", c.in, err)
|
||||
continue
|
||||
}
|
||||
if got != c.want {
|
||||
t.Errorf("parseHumanSize(%q) = %d, want %d", c.in, got, c.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -239,6 +308,135 @@ func TestShippedExampleIsValid(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestFromJobWiring(t *testing.T) {
|
||||
content := "version: 1\njobs:\n" +
|
||||
" producer:\n command: convert\n" +
|
||||
" inputs:\n - path: a.dbml\n format: dbml\n" +
|
||||
" output:\n format: json\n path: build/schema.json\n" +
|
||||
" consumer:\n command: convert\n" +
|
||||
" inputs:\n - from_job: producer\n" +
|
||||
" output:\n format: yaml\n path: build/schema.yaml\n"
|
||||
set := loadOne(t, content)
|
||||
if err := set.Validate(); err != nil {
|
||||
t.Fatalf("validate: %v", err)
|
||||
}
|
||||
plan, err := set.Plan("consumer", true)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(plan) != 2 || plan[0].Name != "producer" || plan[1].Name != "consumer" {
|
||||
t.Fatalf("plan = %v, want [producer consumer]", plan)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFromJobRejectsNonProducer(t *testing.T) {
|
||||
content := "version: 1\njobs:\n" +
|
||||
" lister:\n command: scripts-list\n script_dirs: [migrations]\n" +
|
||||
" consumer:\n command: convert\n" +
|
||||
" inputs:\n - from_job: lister\n" +
|
||||
" output:\n format: yaml\n path: out.yaml\n"
|
||||
set := loadOne(t, content)
|
||||
if err := set.Validate(); err == nil || !strings.Contains(err.Error(), "from_job") {
|
||||
t.Fatalf("want from_job producer error, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFromJobRejectsUnknownJob(t *testing.T) {
|
||||
content := "version: 1\njobs:\n" +
|
||||
" consumer:\n command: convert\n" +
|
||||
" inputs:\n - from_job: ghost\n" +
|
||||
" output:\n format: yaml\n path: out.yaml\n"
|
||||
set := loadOne(t, content)
|
||||
if err := set.Validate(); err == nil || !strings.Contains(err.Error(), "unknown job") {
|
||||
t.Fatalf("want unknown job error, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFromJobCycleDetected(t *testing.T) {
|
||||
content := "version: 1\njobs:\n" +
|
||||
" a:\n command: convert\n" +
|
||||
" inputs:\n - from_job: b\n" +
|
||||
" output:\n format: json\n path: a.json\n" +
|
||||
" b:\n command: convert\n" +
|
||||
" inputs:\n - from_job: a\n" +
|
||||
" output:\n format: json\n path: b.json\n"
|
||||
set := loadOne(t, content)
|
||||
if err := set.Validate(); err == nil || !strings.Contains(err.Error(), "cycle") {
|
||||
t.Fatalf("want cycle error, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSplitJobValidation(t *testing.T) {
|
||||
set := loadOne(t, "version: 1\njobs:\n s:\n command: split\n"+
|
||||
" inputs:\n - path: a.dbml\n format: dbml\n")
|
||||
if err := set.Validate(); err == nil || !strings.Contains(err.Error(), "missing output") {
|
||||
t.Fatalf("want missing output, got %v", err)
|
||||
}
|
||||
set = loadOne(t, "version: 1\njobs:\n s:\n command: split\n"+
|
||||
" inputs:\n - path: a.dbml\n format: dbml\n"+
|
||||
" select:\n tables: [users]\n"+
|
||||
" output:\n format: json\n path: out.json\n")
|
||||
if err := set.Validate(); err != nil {
|
||||
t.Fatalf("expected valid split job, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestInspectJobValidation(t *testing.T) {
|
||||
set := loadOne(t, "version: 1\njobs:\n i:\n command: inspect\n"+
|
||||
" inputs:\n - path: a.dbml\n format: dbml\n")
|
||||
if err := set.Validate(); err == nil || !strings.Contains(err.Error(), "report") {
|
||||
t.Fatalf("want report required, got %v", err)
|
||||
}
|
||||
set = loadOne(t, "version: 1\njobs:\n i:\n command: inspect\n"+
|
||||
" inputs:\n - path: a.dbml\n format: dbml\n"+
|
||||
" report:\n format: json\n path: build/report.json\n")
|
||||
if err := set.Validate(); err != nil {
|
||||
t.Fatalf("expected valid inspect job, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDiffJobValidation(t *testing.T) {
|
||||
set := loadOne(t, "version: 1\njobs:\n d:\n command: diff\n"+
|
||||
" inputs:\n - path: a.dbml\n format: dbml\n"+
|
||||
" report:\n format: summary\n")
|
||||
if err := set.Validate(); err == nil || !strings.Contains(err.Error(), "exactly 2 inputs") {
|
||||
t.Fatalf("want exactly 2 inputs, got %v", err)
|
||||
}
|
||||
set = loadOne(t, "version: 1\njobs:\n d:\n command: diff\n"+
|
||||
" inputs:\n - path: a.dbml\n format: dbml\n"+
|
||||
" - path: b.dbml\n format: dbml\n"+
|
||||
" report:\n format: summary\n")
|
||||
if err := set.Validate(); err != nil {
|
||||
t.Fatalf("expected valid diff job, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestScriptsExecValidation(t *testing.T) {
|
||||
set := loadOne(t, "version: 1\njobs:\n x:\n command: scripts-exec\n"+
|
||||
" script_dirs: [migrations]\n")
|
||||
if err := set.Validate(); err == nil || !strings.Contains(err.Error(), "conn_env") {
|
||||
t.Fatalf("want output.conn_env required, got %v", err)
|
||||
}
|
||||
set = loadOne(t, "version: 1\njobs:\n x:\n command: scripts-exec\n"+
|
||||
" script_dirs: [migrations]\n"+
|
||||
" output:\n conn_env: TARGET_DB_URL\n")
|
||||
if err := set.Validate(); err != nil {
|
||||
t.Fatalf("expected valid scripts-exec job, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSafeJoinRejectsSymlinkEscape(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
outside := t.TempDir()
|
||||
link := filepath.Join(root, "link")
|
||||
if err := os.Symlink(outside, link); err != nil {
|
||||
t.Skipf("symlink not supported: %v", err)
|
||||
}
|
||||
if _, err := SafeJoin(root, "link/x.sql"); err == nil {
|
||||
t.Fatal("expected rejection of a path escaping via a symlink")
|
||||
}
|
||||
}
|
||||
|
||||
func TestScriptsListValidation(t *testing.T) {
|
||||
set := loadOne(t, "version: 1\njobs:\n s:\n command: scripts-list\n")
|
||||
if err := set.Validate(); err == nil || !strings.Contains(err.Error(), "script_dir") {
|
||||
|
||||
Reference in New Issue
Block a user