warkanum
495a21b67b
Implements tests/_plans and previously deferred packages; updates plan README with new coverage numbers.
263 lines
15 KiB
Go
263 lines
15 KiB
Go
package jobs
|
|
|
|
import (
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
"testing"
|
|
)
|
|
|
|
// validateYAML loads one job file and returns the Validate error text ("" when valid).
|
|
func validateYAML(t *testing.T, body string) string {
|
|
t.Helper()
|
|
set := loadOne(t, "version: 1\njobs:\n"+body)
|
|
if err := set.Validate(); err != nil {
|
|
return err.Error()
|
|
}
|
|
return ""
|
|
}
|
|
|
|
func TestValidateJobTable(t *testing.T) {
|
|
in := " inputs:\n - path: a.dbml\n format: dbml\n"
|
|
out := " output:\n format: json\n path: out.json\n"
|
|
tests := []struct {
|
|
name string
|
|
job string
|
|
want string // substring of the error, "" for valid
|
|
}{
|
|
{"missing command", " x:\n description: d\n", "missing command"},
|
|
{"convert valid", " x:\n command: convert\n" + in + out, ""},
|
|
{"convert script dirs", " x:\n command: convert\n script_dirs: [s]\n" + in + out, "script_dirs is not valid"},
|
|
{"convert missing output", " x:\n command: convert\n" + in, "missing output"},
|
|
{"convert output missing format", " x:\n command: convert\n" + in + " output:\n path: o\n", "output: missing format"},
|
|
{"convert output unsupported format", " x:\n command: convert\n" + in + " output:\n format: nope\n path: o\n", "unsupported output format"},
|
|
{"convert output missing path", " x:\n command: convert\n" + in + " output:\n format: json\n", "output: missing path"},
|
|
{"convert output conn_env on non-exec format", " x:\n command: convert\n" + in + " output:\n format: json\n conn_env: DB\n", "not supported for format"},
|
|
{"convert output path and conn_env", " x:\n command: convert\n" + in + " output:\n format: pgsql\n conn_env: DB\n path: o.sql\n", "either path or conn_env"},
|
|
{"convert output conn_env ok", " x:\n command: convert\n" + in + " output:\n format: pgsql\n conn_env: DB\n", ""},
|
|
{"output secret conn_env", " x:\n command: convert\n" + in + " output:\n format: pgsql\n conn_env: postgres://u:p@h/db\n", "environment variable name"},
|
|
{"merge needs two inputs", " x:\n command: merge\n" + in + out, "at least 2 input"},
|
|
{"input missing format", " x:\n command: convert\n inputs:\n - path: a\n" + out, "missing format"},
|
|
{"input unsupported format", " x:\n command: convert\n inputs:\n - path: a\n format: nope\n" + out, "unsupported input format"},
|
|
{"input file missing path", " x:\n command: convert\n inputs:\n - format: dbml\n" + out, "missing path"},
|
|
{"input file with conn_env", " x:\n command: convert\n inputs:\n - path: a\n format: dbml\n conn_env: DB\n" + out, "does not use conn_env"},
|
|
{"input db missing conn_env", " x:\n command: convert\n inputs:\n - format: pgsql\n" + out, "requires conn_env"},
|
|
{"input db with path", " x:\n command: convert\n inputs:\n - format: pgsql\n conn_env: DB\n path: a\n" + out, "takes conn_env, not path"},
|
|
{"input db ok", " x:\n command: convert\n inputs:\n - format: pgsql\n conn_env: DB\n" + out, ""},
|
|
{"input secret conn_env", " x:\n command: convert\n inputs:\n - format: pgsql\n conn_env: \"host=h password=p\"\n" + out, "environment variable name"},
|
|
{"bad log size", " x:\n command: convert\n log_max_size: lots\n" + in + out, "log_max_size"},
|
|
{"absolute logfile", " x:\n command: convert\n logfile: /var/log/x.log\n" + in + out, "absolute paths"},
|
|
{"home path", " x:\n command: convert\n template: ~/t\n" + in + out, "home-relative"},
|
|
{"report path traversal", " x:\n command: inspect\n" + in + " report:\n format: json\n path: ../r.json\n", "escapes"},
|
|
{"script_dir traversal", " x:\n command: scripts-list\n script_dirs: [../x]\n", "escapes"},
|
|
|
|
{"templ valid", " x:\n command: templ\n" + in + " template: t.tmpl\n mode: table\n output:\n format: text\n path: o\n", ""},
|
|
{"templ pgsql input valid", " x:\n command: templ\n inputs:\n - format: pgsql\n conn_env: DB\n template: t.tmpl\n", ""},
|
|
{"templ no inputs", " x:\n command: templ\n template: t.tmpl\n", "at least 1 input"},
|
|
{"templ no template", " x:\n command: templ\n" + in, "requires template"},
|
|
{"templ bad mode", " x:\n command: templ\n" + in + " template: t\n mode: weird\n", "unsupported mode"},
|
|
{"templ script dirs", " x:\n command: templ\n" + in + " template: t\n script_dirs: [s]\n", "script_dirs is not valid"},
|
|
{"templ db output", " x:\n command: templ\n" + in + " template: t\n output:\n conn_env: DB\n", "does not support database output"},
|
|
{"templ non-text output", " x:\n command: templ\n" + in + " template: t\n output:\n format: json\n path: o\n", "only output.format: text"},
|
|
{"templ input missing format", " x:\n command: templ\n inputs:\n - path: a\n template: t\n", "missing format"},
|
|
{"templ pgsql input without conn_env", " x:\n command: templ\n inputs:\n - format: pgsql\n template: t\n", "requires conn_env"},
|
|
{"templ pgsql input with path", " x:\n command: templ\n inputs:\n - format: pgsql\n conn_env: DB\n path: a\n template: t\n", "takes conn_env, not path"},
|
|
{"templ file input without path", " x:\n command: templ\n inputs:\n - format: dbml\n template: t\n", "missing path"},
|
|
{"templ file input with conn_env", " x:\n command: templ\n inputs:\n - path: a\n format: dbml\n conn_env: DB\n template: t\n", "does not use conn_env"},
|
|
{"templ unsupported input format", " x:\n command: templ\n inputs:\n - path: a\n format: nope\n template: t\n", "unsupported templ input format"},
|
|
{"templ secret conn_env", " x:\n command: templ\n inputs:\n - format: pgsql\n conn_env: a/b\n template: t\n", "environment variable name"},
|
|
|
|
{"split needs input", " x:\n command: split\n" + out, "at least 1 input"},
|
|
{"split script dirs", " x:\n command: split\n" + in + " script_dirs: [s]\n" + out, "script_dirs is not valid"},
|
|
{"split report", " x:\n command: split\n" + in + " report:\n format: json\n path: r\n" + out, "report is not valid"},
|
|
{"split db output", " x:\n command: split\n" + in + " output:\n format: pgsql\n conn_env: DB\n", "writes a file"},
|
|
|
|
{"inspect script dirs", " x:\n command: inspect\n" + in + " script_dirs: [s]\n report:\n path: r\n", "script_dirs is not valid"},
|
|
{"inspect output", " x:\n command: inspect\n" + in + out + " report:\n path: r\n", "output is not valid"},
|
|
{"inspect bad report format", " x:\n command: inspect\n" + in + " report:\n format: html\n path: r\n", "not supported"},
|
|
{"inspect report without path", " x:\n command: inspect\n" + in + " report:\n format: json\n", "requires report.path"},
|
|
{"inspect default format ok", " x:\n command: inspect\n" + in + " report:\n path: r.md\n", ""},
|
|
{"diff summary without path ok", " x:\n command: diff\n" + in + " - path: b.dbml\n format: dbml\n report:\n format: summary\n", ""},
|
|
{"diff json needs path", " x:\n command: diff\n" + in + " - path: b.dbml\n format: dbml\n report:\n format: json\n", "requires report.path"},
|
|
{"diff output", " x:\n command: diff\n" + in + " - path: b.dbml\n format: dbml\n" + out + " report:\n format: summary\n", "output is not valid"},
|
|
{"diff script dirs", " x:\n command: diff\n" + in + " - path: b.dbml\n format: dbml\n script_dirs: [s]\n report:\n format: summary\n", "script_dirs is not valid"},
|
|
{"diff no report", " x:\n command: diff\n" + in + " - path: b.dbml\n format: dbml\n", "requires a report block"},
|
|
|
|
{"scripts-list inputs", " x:\n command: scripts-list\n script_dirs: [s]\n" + in, "inputs is not valid"},
|
|
{"scripts-list output", " x:\n command: scripts-list\n script_dirs: [s]\n" + out, "output is not valid"},
|
|
{"scripts-exec inputs", " x:\n command: scripts-exec\n script_dirs: [s]\n" + in + " output:\n conn_env: DB\n", "inputs is not valid"},
|
|
{"scripts-exec report", " x:\n command: scripts-exec\n script_dirs: [s]\n report:\n path: r\n output:\n conn_env: DB\n", "report is not valid"},
|
|
{"scripts-exec output path", " x:\n command: scripts-exec\n script_dirs: [s]\n output:\n conn_env: DB\n path: p\n", "output.path is not supported"},
|
|
{"scripts-exec non-pgsql", " x:\n command: scripts-exec\n script_dirs: [s]\n output:\n conn_env: DB\n format: mssql\n", "only supports pgsql"},
|
|
{"scripts-exec secret conn_env", " x:\n command: scripts-exec\n script_dirs: [s]\n output:\n conn_env: \"postgres://u@h/d\"\n", "environment variable name"},
|
|
{"scripts-exec no script dirs", " x:\n command: scripts-exec\n output:\n conn_env: DB\n", "requires at least one script_dir"},
|
|
{"scripts-exec pgsql format ok", " x:\n command: scripts-exec\n script_dirs: [s]\n output:\n conn_env: DB\n format: pgsql\n", ""},
|
|
}
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
got := validateYAML(t, tt.job)
|
|
if tt.want == "" {
|
|
if got != "" {
|
|
t.Errorf("expected valid, got: %s", got)
|
|
}
|
|
return
|
|
}
|
|
if !strings.Contains(got, tt.want) {
|
|
t.Errorf("error %q does not contain %q", got, tt.want)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestFromJobInputShape(t *testing.T) {
|
|
producer := " p:\n command: convert\n inputs:\n - path: a.dbml\n format: dbml\n output:\n format: json\n path: out.json\n"
|
|
tests := []struct {
|
|
name string
|
|
input string
|
|
want string
|
|
}{
|
|
{"path", " - from_job: p\n path: x\n", "takes no path"},
|
|
{"format", " - from_job: p\n format: json\n", "drop format"},
|
|
{"conn_env", " - from_job: p\n conn_env: DB\n", "takes no conn_env"},
|
|
}
|
|
for _, tt := range tests {
|
|
for _, cmd := range []string{"convert", "templ"} {
|
|
t.Run(cmd+"/"+tt.name, func(t *testing.T) {
|
|
extra := " output:\n format: json\n path: o.json\n"
|
|
if cmd == "templ" {
|
|
extra = " template: t.tmpl\n"
|
|
}
|
|
got := validateYAML(t, producer+" c:\n command: "+cmd+"\n inputs:\n"+tt.input+extra)
|
|
if !strings.Contains(got, tt.want) {
|
|
t.Errorf("error %q does not contain %q", got, tt.want)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestResolvedLogPolicy(t *testing.T) {
|
|
keep2 := 2
|
|
keep0 := 0
|
|
tests := []struct {
|
|
name string
|
|
job Job
|
|
want LogPolicy
|
|
}{
|
|
{"built-in defaults", Job{}, LogPolicy{MaxSizeBytes: defaultLogMaxSizeBytes, Keep: defaultLogKeep}},
|
|
{"file defaults", Job{fileDefaults: &Defaults{LogMaxSize: "1MB", LogKeep: 7}}, LogPolicy{MaxSizeBytes: 1 << 20, Keep: 7}},
|
|
{"file defaults invalid size falls back", Job{fileDefaults: &Defaults{LogMaxSize: "junk", LogKeep: 0}}, LogPolicy{MaxSizeBytes: defaultLogMaxSizeBytes, Keep: defaultLogKeep}},
|
|
{"job overrides file", Job{fileDefaults: &Defaults{LogMaxSize: "1MB", LogKeep: 7}, LogMaxSize: "2kb", LogKeep: &keep2}, LogPolicy{MaxSizeBytes: 2 << 10, Keep: 2}},
|
|
{"job keep zero is honoured", Job{LogKeep: &keep0}, LogPolicy{MaxSizeBytes: defaultLogMaxSizeBytes, Keep: 0}},
|
|
{"job invalid size ignored", Job{LogMaxSize: "junk"}, LogPolicy{MaxSizeBytes: defaultLogMaxSizeBytes, Keep: defaultLogKeep}},
|
|
}
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
if got := tt.job.ResolvedLogPolicy(); got != tt.want {
|
|
t.Errorf("got %+v, want %+v", got, tt.want)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestLoadAppliesFileDefaultsAndDir(t *testing.T) {
|
|
dir := t.TempDir()
|
|
p := filepath.Join(dir, "relspec.yml")
|
|
write(t, p, "version: 1\ndefaults:\n log_max_size: 1MB\n log_keep: 9\n"+"jobs:\n a:\n command: convert\n inputs:\n - path: a.dbml\n format: dbml\n output:\n format: json\n path: o.json\n")
|
|
set, err := Load([]string{p})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
job := set.Jobs["a"]
|
|
if job.Dir() != dir {
|
|
t.Errorf("Dir = %q, want %q", job.Dir(), dir)
|
|
}
|
|
if pol := job.ResolvedLogPolicy(); pol.MaxSizeBytes != 1<<20 || pol.Keep != 9 {
|
|
t.Errorf("policy %+v", pol)
|
|
}
|
|
}
|
|
|
|
func TestSetNamesSorted(t *testing.T) {
|
|
set := &Set{Jobs: map[string]*Job{"b": {}, "a": {}, "c": {}}}
|
|
if got := strings.Join(set.Names(), ","); got != "a,b,c" {
|
|
t.Errorf("got %s", got)
|
|
}
|
|
}
|
|
|
|
func TestPlanErrors(t *testing.T) {
|
|
set := loadOne(t, "version: 1\njobs:\n"+
|
|
" a:\n command: convert\n depends_on: [ghost]\n inputs:\n - path: a.dbml\n format: dbml\n output:\n format: json\n path: o.json\n"+
|
|
" b:\n command: convert\n inputs:\n - path: a.dbml\n format: dbml\n output:\n format: json\n path: o2.json\n")
|
|
|
|
if _, err := set.Plan("nope", true); err == nil || !strings.Contains(err.Error(), "unknown job") || !strings.Contains(err.Error(), "a, b") {
|
|
t.Errorf("unknown job: %v", err)
|
|
}
|
|
if _, err := set.Plan("a", true); err == nil || !strings.Contains(err.Error(), "unknown job \"ghost\"") {
|
|
t.Errorf("unknown dependency: %v", err)
|
|
}
|
|
// Without dependencies the declared dependency is not walked.
|
|
if got, err := set.Plan("a", false); err != nil || len(got) != 1 || got[0].Name != "a" {
|
|
t.Errorf("no-deps plan: %v %v", got, err)
|
|
}
|
|
}
|
|
|
|
func TestPlanCycleAtRuntime(t *testing.T) {
|
|
set := &Set{Jobs: map[string]*Job{
|
|
"a": {Name: "a", DependsOn: []string{"b"}},
|
|
"b": {Name: "b", DependsOn: []string{"a"}},
|
|
}}
|
|
if _, err := set.Plan("a", true); err == nil || !strings.Contains(err.Error(), "cycle") {
|
|
t.Errorf("want cycle error, got %v", err)
|
|
}
|
|
}
|
|
|
|
func TestDiscoverErrorsAndFiltering(t *testing.T) {
|
|
if _, err := Discover(filepath.Join(t.TempDir(), "missing")); err == nil {
|
|
t.Error("missing dir must fail")
|
|
}
|
|
dir := t.TempDir()
|
|
for _, f := range []string{"relspec.yaml", "relspec.b.yml", "relspec.a.yaml", "relspec.txt", "other.yml", "relspec"} {
|
|
write(t, filepath.Join(dir, f), "")
|
|
}
|
|
if err := os.Mkdir(filepath.Join(dir, "relspec.dir.yml"), 0o755); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
got, err := Discover(dir)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var names []string
|
|
for _, p := range got {
|
|
names = append(names, filepath.Base(p))
|
|
}
|
|
if strings.Join(names, ",") != "relspec.yaml,relspec.a.yaml,relspec.b.yml" {
|
|
t.Errorf("got %v", names)
|
|
}
|
|
}
|
|
|
|
func TestSafeJoinCases(t *testing.T) {
|
|
root := t.TempDir()
|
|
if got, err := SafeJoin(root, "sub/file.sql"); err != nil || !strings.HasSuffix(got, filepath.Join("sub", "file.sql")) {
|
|
t.Errorf("nested: %q %v", got, err)
|
|
}
|
|
for _, bad := range []string{"", "/etc/passwd", "~/x", "..", "../x", "a/../../x"} {
|
|
if _, err := SafeJoin(root, bad); err == nil {
|
|
t.Errorf("SafeJoin(%q) must fail", bad)
|
|
}
|
|
}
|
|
if _, err := SafeJoin(filepath.Join(root, "does", "not", "exist"), "x"); err == nil {
|
|
t.Error("unresolvable root must fail")
|
|
}
|
|
}
|
|
|
|
func TestLooksLikeSecret(t *testing.T) {
|
|
for in, want := range map[string]bool{
|
|
"": false, "DB_URL": false, "MY_DB": false,
|
|
"postgres://u:p@h/db": true, "host=h": true, "a b": true, "a/b": true, "u@h": true, "k:v": true,
|
|
} {
|
|
if got := looksLikeSecret(in); got != want {
|
|
t.Errorf("looksLikeSecret(%q) = %v, want %v", in, got, want)
|
|
}
|
|
}
|
|
}
|