Hein
84a6b31873
Implements the remaining items from issue #20: - version is now forward-permissive: any value >= 1 is accepted; a newer-than-known version loads best-effort (unknown fields ignored, warning printed) instead of hard-failing on "must be 1" - from_job input reference: `inputs: [{ from_job: <job> }]` resolves to that job's single-file output + format and implies a dependency edge; combined depends_on + from_job graph gets topological ordering and cycle detection - logfile size-rotation, on by default (5MB, keep 3), overridable per job (log_max_size / log_keep) or file-wide via a top-level defaults block - new commands: split (schema/table subsetting via select:), inspect (rule validation -> markdown/json report, fails job on enforced-rule errors), diff (compare exactly two schemas, never fails), scripts-exec (run SQL script dirs against a live PostgreSQL database) - atomic single-file output/report writes (temp file + rename) - symlink-escape hardening in SafeJoin via EvalSymlinks preflight Updates docs/JOB_FILES.md and examples/jobs/relspec.yml accordingly.
450 lines
15 KiB
Go
450 lines
15 KiB
Go
package jobs
|
|
|
|
import (
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
"testing"
|
|
)
|
|
|
|
func write(t *testing.T, path, content string) {
|
|
t.Helper()
|
|
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := os.WriteFile(path, []byte(content), 0o644); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
|
|
func TestDiscoverDeterministicOrder(t *testing.T) {
|
|
dir := t.TempDir()
|
|
for _, n := range []string{
|
|
"relspec.yml", "relspec.zeta.yml", "relspec.alpha.yaml",
|
|
"relspec.beta.yml", "notes.yml", "relspec.txt",
|
|
} {
|
|
write(t, filepath.Join(dir, n), "version: 1\njobs: {}\n")
|
|
}
|
|
got, err := Discover(dir)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var bases []string
|
|
for _, p := range got {
|
|
bases = append(bases, filepath.Base(p))
|
|
}
|
|
want := []string{"relspec.yml", "relspec.alpha.yaml", "relspec.beta.yml", "relspec.zeta.yml"}
|
|
if strings.Join(bases, ",") != strings.Join(want, ",") {
|
|
t.Fatalf("discover order = %v, want %v", bases, want)
|
|
}
|
|
|
|
// Second call must return the identical order.
|
|
got2, _ := Discover(dir)
|
|
for i := range got {
|
|
if got[i] != got2[i] {
|
|
t.Fatalf("discover not deterministic: %v vs %v", got, got2)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestLoadRejectsUnknownFields(t *testing.T) {
|
|
dir := t.TempDir()
|
|
p := filepath.Join(dir, "relspec.yml")
|
|
write(t, p, "version: 1\njobs:\n a:\n command: convert\n bogus: true\n")
|
|
if _, err := Load([]string{p}); err == nil {
|
|
t.Fatal("expected error for unknown field")
|
|
}
|
|
}
|
|
|
|
func TestLoadWarnsOnNewerVersion(t *testing.T) {
|
|
dir := t.TempDir()
|
|
p := filepath.Join(dir, "relspec.yml")
|
|
// A newer version loads best-effort with a warning, and unknown fields
|
|
// from the newer schema are ignored rather than rejected.
|
|
write(t, p, "version: 99\njobs:\n a:\n command: convert\n"+
|
|
" inputs:\n - path: a.dbml\n format: dbml\n"+
|
|
" output:\n format: json\n path: out.json\n"+
|
|
" future_field: whatever\n")
|
|
set, err := Load([]string{p})
|
|
if err != nil {
|
|
t.Fatalf("newer version should load, got %v", err)
|
|
}
|
|
if len(set.Warnings) == 0 {
|
|
t.Fatal("expected a warning about the newer version")
|
|
}
|
|
if err := set.Validate(); err != nil {
|
|
t.Fatalf("validate: %v", err)
|
|
}
|
|
}
|
|
|
|
func TestLoadAcceptsOmittedVersion(t *testing.T) {
|
|
dir := t.TempDir()
|
|
p := filepath.Join(dir, "relspec.yml")
|
|
write(t, p, "jobs:\n a:\n command: convert\n"+
|
|
" inputs:\n - path: a.dbml\n format: dbml\n"+
|
|
" output:\n format: json\n path: out.json\n")
|
|
set, err := Load([]string{p})
|
|
if err != nil {
|
|
t.Fatalf("omitted version should load, got %v", err)
|
|
}
|
|
if len(set.Warnings) != 0 {
|
|
t.Fatalf("omitted version should not warn, got %v", set.Warnings)
|
|
}
|
|
}
|
|
|
|
func TestLoadStillRejectsUnknownFieldsAtCurrentVersion(t *testing.T) {
|
|
dir := t.TempDir()
|
|
p := filepath.Join(dir, "relspec.yml")
|
|
write(t, p, "version: 1\njobs:\n a:\n command: convert\n bogus: true\n")
|
|
if _, err := Load([]string{p}); err == nil {
|
|
t.Fatal("expected unknown-field rejection at the current version")
|
|
}
|
|
}
|
|
|
|
func TestParseHumanSize(t *testing.T) {
|
|
cases := []struct {
|
|
in string
|
|
want int64
|
|
bad bool
|
|
}{
|
|
{"", 0, false},
|
|
{"512", 512, false},
|
|
{"512B", 512, false},
|
|
{"1KB", 1 << 10, false},
|
|
{"5MB", 5 << 20, false},
|
|
{"1gb", 1 << 30, false},
|
|
{" 2 MB ", 2 << 20, false},
|
|
{"nonsense", 0, true},
|
|
{"-1MB", 0, true},
|
|
}
|
|
for _, c := range cases {
|
|
got, err := parseHumanSize(c.in)
|
|
if c.bad {
|
|
if err == nil {
|
|
t.Errorf("parseHumanSize(%q): expected error", c.in)
|
|
}
|
|
continue
|
|
}
|
|
if err != nil {
|
|
t.Errorf("parseHumanSize(%q): %v", c.in, err)
|
|
continue
|
|
}
|
|
if got != c.want {
|
|
t.Errorf("parseHumanSize(%q) = %d, want %d", c.in, got, c.want)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestLoadRejectsDuplicateJobAcrossFiles(t *testing.T) {
|
|
dir := t.TempDir()
|
|
a := filepath.Join(dir, "relspec.yml")
|
|
b := filepath.Join(dir, "relspec.extra.yml")
|
|
write(t, a, jobFileConvert("build"))
|
|
write(t, b, jobFileConvert("build"))
|
|
_, err := Load([]string{a, b})
|
|
if err == nil || !strings.Contains(err.Error(), "duplicate job") {
|
|
t.Fatalf("expected duplicate job error, got %v", err)
|
|
}
|
|
}
|
|
|
|
func jobFileConvert(name string) string {
|
|
return "version: 1\njobs:\n " + name + ":\n command: convert\n" +
|
|
" inputs:\n - path: a.dbml\n format: dbml\n" +
|
|
" output:\n format: json\n path: out.json\n"
|
|
}
|
|
|
|
func loadOne(t *testing.T, content string) *Set {
|
|
t.Helper()
|
|
dir := t.TempDir()
|
|
p := filepath.Join(dir, "relspec.yml")
|
|
write(t, p, content)
|
|
set, err := Load([]string{p})
|
|
if err != nil {
|
|
t.Fatalf("load: %v", err)
|
|
}
|
|
return set
|
|
}
|
|
|
|
func TestValidateUnknownCommand(t *testing.T) {
|
|
set := loadOne(t, "version: 1\njobs:\n x:\n command: rm-rf\n")
|
|
err := set.Validate()
|
|
if err == nil || !strings.Contains(err.Error(), "unsupported command") {
|
|
t.Fatalf("want unsupported command, got %v", err)
|
|
}
|
|
}
|
|
|
|
func TestValidateShellStringCommandRejected(t *testing.T) {
|
|
set := loadOne(t, "version: 1\njobs:\n x:\n command: \"bash -c 'echo hi'\"\n")
|
|
if err := set.Validate(); err == nil {
|
|
t.Fatal("expected arbitrary shell command to be rejected")
|
|
}
|
|
}
|
|
|
|
func TestValidateMissingInputs(t *testing.T) {
|
|
set := loadOne(t, "version: 1\njobs:\n x:\n command: convert\n output:\n format: json\n path: o.json\n")
|
|
if err := set.Validate(); err == nil || !strings.Contains(err.Error(), "at least 1 input") {
|
|
t.Fatalf("want missing input error, got %v", err)
|
|
}
|
|
}
|
|
|
|
func TestValidateUnknownFormat(t *testing.T) {
|
|
set := loadOne(t, "version: 1\njobs:\n x:\n command: convert\n"+
|
|
" inputs:\n - path: a.xyz\n format: xyz\n"+
|
|
" output:\n format: json\n path: o.json\n")
|
|
if err := set.Validate(); err == nil || !strings.Contains(err.Error(), "unsupported input format") {
|
|
t.Fatalf("want unsupported input format, got %v", err)
|
|
}
|
|
}
|
|
|
|
func TestValidatePathTraversalRejected(t *testing.T) {
|
|
cases := []string{"../secret.dbml", "/etc/passwd", "~/x.dbml", "a/../../b.dbml"}
|
|
for _, bad := range cases {
|
|
set := loadOne(t, "version: 1\njobs:\n x:\n command: convert\n"+
|
|
" inputs:\n - path: \""+bad+"\"\n format: dbml\n"+
|
|
" output:\n format: json\n path: o.json\n")
|
|
if err := set.Validate(); err == nil {
|
|
t.Fatalf("path %q: expected rejection", bad)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestValidateOutputTraversalRejected(t *testing.T) {
|
|
set := loadOne(t, "version: 1\njobs:\n x:\n command: convert\n"+
|
|
" inputs:\n - path: a.dbml\n format: dbml\n"+
|
|
" output:\n format: json\n path: ../../evil.json\n")
|
|
if err := set.Validate(); err == nil {
|
|
t.Fatal("expected output path traversal rejection")
|
|
}
|
|
}
|
|
|
|
func TestValidateConnEnvMustBeName(t *testing.T) {
|
|
set := loadOne(t, "version: 1\njobs:\n x:\n command: convert\n"+
|
|
" inputs:\n - format: pgsql\n conn_env: \"postgres://u:p@h/db\"\n"+
|
|
" output:\n format: json\n path: o.json\n")
|
|
if err := set.Validate(); err == nil || !strings.Contains(err.Error(), "environment variable name") {
|
|
t.Fatalf("want conn_env name error, got %v", err)
|
|
}
|
|
}
|
|
|
|
func TestValidateDependsOnUnknown(t *testing.T) {
|
|
set := loadOne(t, "version: 1\njobs:\n x:\n command: convert\n depends_on: [nope]\n"+
|
|
" inputs:\n - path: a.dbml\n format: dbml\n"+
|
|
" output:\n format: json\n path: o.json\n")
|
|
if err := set.Validate(); err == nil || !strings.Contains(err.Error(), "unknown job") {
|
|
t.Fatalf("want unknown dependency error, got %v", err)
|
|
}
|
|
}
|
|
|
|
func TestValidateDependencyCycle(t *testing.T) {
|
|
content := "version: 1\njobs:\n" +
|
|
jobBlock("a", "b") + jobBlock("b", "c") + jobBlock("c", "a")
|
|
set := loadOne(t, content)
|
|
err := set.Validate()
|
|
if err == nil || !strings.Contains(err.Error(), "cycle") {
|
|
t.Fatalf("want cycle error, got %v", err)
|
|
}
|
|
}
|
|
|
|
func jobBlock(name, dep string) string {
|
|
return " " + name + ":\n command: convert\n depends_on: [" + dep + "]\n" +
|
|
" inputs:\n - path: a.dbml\n format: dbml\n" +
|
|
" output:\n format: json\n path: " + name + ".json\n"
|
|
}
|
|
|
|
func TestPlanTopologicalOrder(t *testing.T) {
|
|
content := "version: 1\njobs:\n" +
|
|
" base:\n command: convert\n inputs:\n - path: a.dbml\n format: dbml\n output:\n format: json\n path: base.json\n" +
|
|
" mid:\n command: convert\n depends_on: [base]\n inputs:\n - path: a.dbml\n format: dbml\n output:\n format: json\n path: mid.json\n" +
|
|
" top:\n command: convert\n depends_on: [mid]\n inputs:\n - path: a.dbml\n format: dbml\n output:\n format: json\n path: top.json\n"
|
|
set := loadOne(t, content)
|
|
if err := set.Validate(); err != nil {
|
|
t.Fatalf("validate: %v", err)
|
|
}
|
|
plan, err := set.Plan("top", true)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var order []string
|
|
for _, j := range plan {
|
|
order = append(order, j.Name)
|
|
}
|
|
if strings.Join(order, ",") != "base,mid,top" {
|
|
t.Fatalf("plan order = %v, want [base mid top]", order)
|
|
}
|
|
|
|
solo, err := set.Plan("top", false)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(solo) != 1 || solo[0].Name != "top" {
|
|
t.Fatalf("no-deps plan = %v, want [top]", solo)
|
|
}
|
|
}
|
|
|
|
func TestSafeJoinStaysInsideRoot(t *testing.T) {
|
|
root := t.TempDir()
|
|
if _, err := SafeJoin(root, "sub/dir/file.sql"); err != nil {
|
|
t.Fatalf("expected ok, got %v", err)
|
|
}
|
|
if _, err := SafeJoin(root, "../escape"); err == nil {
|
|
t.Fatal("expected escape rejection")
|
|
}
|
|
if _, err := SafeJoin(root, "/abs"); err == nil {
|
|
t.Fatal("expected absolute rejection")
|
|
}
|
|
}
|
|
|
|
func TestShippedExampleIsValid(t *testing.T) {
|
|
path := filepath.Join("..", "..", "examples", "jobs", "relspec.yml")
|
|
set, err := Load([]string{path})
|
|
if err != nil {
|
|
t.Fatalf("load example: %v", err)
|
|
}
|
|
if err := set.Validate(); err != nil {
|
|
t.Fatalf("example manifest failed validation: %v", err)
|
|
}
|
|
if _, err := set.Plan("build-json", true); err != nil {
|
|
t.Fatalf("plan example: %v", err)
|
|
}
|
|
}
|
|
|
|
func TestFromJobWiring(t *testing.T) {
|
|
content := "version: 1\njobs:\n" +
|
|
" producer:\n command: convert\n" +
|
|
" inputs:\n - path: a.dbml\n format: dbml\n" +
|
|
" output:\n format: json\n path: build/schema.json\n" +
|
|
" consumer:\n command: convert\n" +
|
|
" inputs:\n - from_job: producer\n" +
|
|
" output:\n format: yaml\n path: build/schema.yaml\n"
|
|
set := loadOne(t, content)
|
|
if err := set.Validate(); err != nil {
|
|
t.Fatalf("validate: %v", err)
|
|
}
|
|
plan, err := set.Plan("consumer", true)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(plan) != 2 || plan[0].Name != "producer" || plan[1].Name != "consumer" {
|
|
t.Fatalf("plan = %v, want [producer consumer]", plan)
|
|
}
|
|
}
|
|
|
|
func TestFromJobRejectsNonProducer(t *testing.T) {
|
|
content := "version: 1\njobs:\n" +
|
|
" lister:\n command: scripts-list\n script_dirs: [migrations]\n" +
|
|
" consumer:\n command: convert\n" +
|
|
" inputs:\n - from_job: lister\n" +
|
|
" output:\n format: yaml\n path: out.yaml\n"
|
|
set := loadOne(t, content)
|
|
if err := set.Validate(); err == nil || !strings.Contains(err.Error(), "from_job") {
|
|
t.Fatalf("want from_job producer error, got %v", err)
|
|
}
|
|
}
|
|
|
|
func TestFromJobRejectsUnknownJob(t *testing.T) {
|
|
content := "version: 1\njobs:\n" +
|
|
" consumer:\n command: convert\n" +
|
|
" inputs:\n - from_job: ghost\n" +
|
|
" output:\n format: yaml\n path: out.yaml\n"
|
|
set := loadOne(t, content)
|
|
if err := set.Validate(); err == nil || !strings.Contains(err.Error(), "unknown job") {
|
|
t.Fatalf("want unknown job error, got %v", err)
|
|
}
|
|
}
|
|
|
|
func TestFromJobCycleDetected(t *testing.T) {
|
|
content := "version: 1\njobs:\n" +
|
|
" a:\n command: convert\n" +
|
|
" inputs:\n - from_job: b\n" +
|
|
" output:\n format: json\n path: a.json\n" +
|
|
" b:\n command: convert\n" +
|
|
" inputs:\n - from_job: a\n" +
|
|
" output:\n format: json\n path: b.json\n"
|
|
set := loadOne(t, content)
|
|
if err := set.Validate(); err == nil || !strings.Contains(err.Error(), "cycle") {
|
|
t.Fatalf("want cycle error, got %v", err)
|
|
}
|
|
}
|
|
|
|
func TestSplitJobValidation(t *testing.T) {
|
|
set := loadOne(t, "version: 1\njobs:\n s:\n command: split\n"+
|
|
" inputs:\n - path: a.dbml\n format: dbml\n")
|
|
if err := set.Validate(); err == nil || !strings.Contains(err.Error(), "missing output") {
|
|
t.Fatalf("want missing output, got %v", err)
|
|
}
|
|
set = loadOne(t, "version: 1\njobs:\n s:\n command: split\n"+
|
|
" inputs:\n - path: a.dbml\n format: dbml\n"+
|
|
" select:\n tables: [users]\n"+
|
|
" output:\n format: json\n path: out.json\n")
|
|
if err := set.Validate(); err != nil {
|
|
t.Fatalf("expected valid split job, got %v", err)
|
|
}
|
|
}
|
|
|
|
func TestInspectJobValidation(t *testing.T) {
|
|
set := loadOne(t, "version: 1\njobs:\n i:\n command: inspect\n"+
|
|
" inputs:\n - path: a.dbml\n format: dbml\n")
|
|
if err := set.Validate(); err == nil || !strings.Contains(err.Error(), "report") {
|
|
t.Fatalf("want report required, got %v", err)
|
|
}
|
|
set = loadOne(t, "version: 1\njobs:\n i:\n command: inspect\n"+
|
|
" inputs:\n - path: a.dbml\n format: dbml\n"+
|
|
" report:\n format: json\n path: build/report.json\n")
|
|
if err := set.Validate(); err != nil {
|
|
t.Fatalf("expected valid inspect job, got %v", err)
|
|
}
|
|
}
|
|
|
|
func TestDiffJobValidation(t *testing.T) {
|
|
set := loadOne(t, "version: 1\njobs:\n d:\n command: diff\n"+
|
|
" inputs:\n - path: a.dbml\n format: dbml\n"+
|
|
" report:\n format: summary\n")
|
|
if err := set.Validate(); err == nil || !strings.Contains(err.Error(), "exactly 2 inputs") {
|
|
t.Fatalf("want exactly 2 inputs, got %v", err)
|
|
}
|
|
set = loadOne(t, "version: 1\njobs:\n d:\n command: diff\n"+
|
|
" inputs:\n - path: a.dbml\n format: dbml\n"+
|
|
" - path: b.dbml\n format: dbml\n"+
|
|
" report:\n format: summary\n")
|
|
if err := set.Validate(); err != nil {
|
|
t.Fatalf("expected valid diff job, got %v", err)
|
|
}
|
|
}
|
|
|
|
func TestScriptsExecValidation(t *testing.T) {
|
|
set := loadOne(t, "version: 1\njobs:\n x:\n command: scripts-exec\n"+
|
|
" script_dirs: [migrations]\n")
|
|
if err := set.Validate(); err == nil || !strings.Contains(err.Error(), "conn_env") {
|
|
t.Fatalf("want output.conn_env required, got %v", err)
|
|
}
|
|
set = loadOne(t, "version: 1\njobs:\n x:\n command: scripts-exec\n"+
|
|
" script_dirs: [migrations]\n"+
|
|
" output:\n conn_env: TARGET_DB_URL\n")
|
|
if err := set.Validate(); err != nil {
|
|
t.Fatalf("expected valid scripts-exec job, got %v", err)
|
|
}
|
|
}
|
|
|
|
func TestSafeJoinRejectsSymlinkEscape(t *testing.T) {
|
|
root := t.TempDir()
|
|
outside := t.TempDir()
|
|
link := filepath.Join(root, "link")
|
|
if err := os.Symlink(outside, link); err != nil {
|
|
t.Skipf("symlink not supported: %v", err)
|
|
}
|
|
if _, err := SafeJoin(root, "link/x.sql"); err == nil {
|
|
t.Fatal("expected rejection of a path escaping via a symlink")
|
|
}
|
|
}
|
|
|
|
func TestScriptsListValidation(t *testing.T) {
|
|
set := loadOne(t, "version: 1\njobs:\n s:\n command: scripts-list\n")
|
|
if err := set.Validate(); err == nil || !strings.Contains(err.Error(), "script_dir") {
|
|
t.Fatalf("want script_dir required error, got %v", err)
|
|
}
|
|
set = loadOne(t, "version: 1\njobs:\n s:\n command: scripts-list\n script_dirs: [migrations, extra]\n")
|
|
if err := set.Validate(); err != nil {
|
|
t.Fatalf("expected valid scripts-list job, got %v", err)
|
|
}
|
|
}
|