feat(job): declarative YAML job files for named relspec workflows
Add `relspec job list` and `relspec job run <name>` driven by YAML job manifests (relspec.yml / relspec.<name>.yml), so multi-file merge and conversion workflows can be expressed declaratively instead of as long shell command lines. v1 contract (see docs/JOB_FILES.md): - `command` is a closed allow-list (convert, merge, scripts-list); no field accepts a shell string or executable path. - Deterministic discovery: default file first, then named files sorted lexically; all files merged into one namespace; duplicate job names across files are a hard error. - Every path resolves relative to the job file's directory; absolute, home-relative and directory-escaping paths are rejected at validation. - Database credentials referenced by env-var name via `conn_env:`; connection strings are never stored and are redacted from logs/plan. - Full validation (version, unknown fields, command/format, per-command input/output shape, path traversal, depends_on targets, dependency cycles) runs before anything is read, written or executed; per-job pre-flight then checks input existence, script dirs, env vars and the output overwrite policy for the whole plan. - `depends_on` closure runs in deterministic topological order; `--no-deps` runs only the named job. - `--dry-run` (alias `--plan`) prints the resolved plan and exits 0 without touching inputs, outputs or databases. - A failing job propagates the underlying non-zero exit status, logs FAILED (never OK), and writes no success marker. pkg/jobs is side-effect free (discovery/parse/validate/plan only); execution adapters live in cmd/relspec/job.go. Includes unit tests for discovery, validation, planning and path safety, plus CLI tests for end-to-end convert/merge, scripts-list across multiple directories, dry-run, dependency chains, exit-code propagation and log redaction. Deferred: live `scripts execute` from jobs, split/inspect/diff/templ commands, job-to-job output wiring, log rotation/retention. Refs #20 Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
SG Command
co-authored by
Claude Sonnet 5
Claude Sonnet 5
parent
4115a11845
commit
4d299fda98
@@ -0,0 +1,251 @@
|
||||
package jobs
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func write(t *testing.T, path, content string) {
|
||||
t.Helper()
|
||||
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(path, []byte(content), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDiscoverDeterministicOrder(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
for _, n := range []string{
|
||||
"relspec.yml", "relspec.zeta.yml", "relspec.alpha.yaml",
|
||||
"relspec.beta.yml", "notes.yml", "relspec.txt",
|
||||
} {
|
||||
write(t, filepath.Join(dir, n), "version: 1\njobs: {}\n")
|
||||
}
|
||||
got, err := Discover(dir)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var bases []string
|
||||
for _, p := range got {
|
||||
bases = append(bases, filepath.Base(p))
|
||||
}
|
||||
want := []string{"relspec.yml", "relspec.alpha.yaml", "relspec.beta.yml", "relspec.zeta.yml"}
|
||||
if strings.Join(bases, ",") != strings.Join(want, ",") {
|
||||
t.Fatalf("discover order = %v, want %v", bases, want)
|
||||
}
|
||||
|
||||
// Second call must return the identical order.
|
||||
got2, _ := Discover(dir)
|
||||
for i := range got {
|
||||
if got[i] != got2[i] {
|
||||
t.Fatalf("discover not deterministic: %v vs %v", got, got2)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadRejectsUnknownFields(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
p := filepath.Join(dir, "relspec.yml")
|
||||
write(t, p, "version: 1\njobs:\n a:\n command: convert\n bogus: true\n")
|
||||
if _, err := Load([]string{p}); err == nil {
|
||||
t.Fatal("expected error for unknown field")
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadRejectsBadVersion(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
p := filepath.Join(dir, "relspec.yml")
|
||||
write(t, p, "version: 2\njobs:\n a:\n command: convert\n")
|
||||
_, err := Load([]string{p})
|
||||
if err == nil || !strings.Contains(err.Error(), "unsupported version") {
|
||||
t.Fatalf("expected unsupported version error, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadRejectsDuplicateJobAcrossFiles(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
a := filepath.Join(dir, "relspec.yml")
|
||||
b := filepath.Join(dir, "relspec.extra.yml")
|
||||
write(t, a, jobFileConvert("build"))
|
||||
write(t, b, jobFileConvert("build"))
|
||||
_, err := Load([]string{a, b})
|
||||
if err == nil || !strings.Contains(err.Error(), "duplicate job") {
|
||||
t.Fatalf("expected duplicate job error, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func jobFileConvert(name string) string {
|
||||
return "version: 1\njobs:\n " + name + ":\n command: convert\n" +
|
||||
" inputs:\n - path: a.dbml\n format: dbml\n" +
|
||||
" output:\n format: json\n path: out.json\n"
|
||||
}
|
||||
|
||||
func loadOne(t *testing.T, content string) *Set {
|
||||
t.Helper()
|
||||
dir := t.TempDir()
|
||||
p := filepath.Join(dir, "relspec.yml")
|
||||
write(t, p, content)
|
||||
set, err := Load([]string{p})
|
||||
if err != nil {
|
||||
t.Fatalf("load: %v", err)
|
||||
}
|
||||
return set
|
||||
}
|
||||
|
||||
func TestValidateUnknownCommand(t *testing.T) {
|
||||
set := loadOne(t, "version: 1\njobs:\n x:\n command: rm-rf\n")
|
||||
err := set.Validate()
|
||||
if err == nil || !strings.Contains(err.Error(), "unsupported command") {
|
||||
t.Fatalf("want unsupported command, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateShellStringCommandRejected(t *testing.T) {
|
||||
set := loadOne(t, "version: 1\njobs:\n x:\n command: \"bash -c 'echo hi'\"\n")
|
||||
if err := set.Validate(); err == nil {
|
||||
t.Fatal("expected arbitrary shell command to be rejected")
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateMissingInputs(t *testing.T) {
|
||||
set := loadOne(t, "version: 1\njobs:\n x:\n command: convert\n output:\n format: json\n path: o.json\n")
|
||||
if err := set.Validate(); err == nil || !strings.Contains(err.Error(), "at least 1 input") {
|
||||
t.Fatalf("want missing input error, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateUnknownFormat(t *testing.T) {
|
||||
set := loadOne(t, "version: 1\njobs:\n x:\n command: convert\n"+
|
||||
" inputs:\n - path: a.xyz\n format: xyz\n"+
|
||||
" output:\n format: json\n path: o.json\n")
|
||||
if err := set.Validate(); err == nil || !strings.Contains(err.Error(), "unsupported input format") {
|
||||
t.Fatalf("want unsupported input format, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidatePathTraversalRejected(t *testing.T) {
|
||||
cases := []string{"../secret.dbml", "/etc/passwd", "~/x.dbml", "a/../../b.dbml"}
|
||||
for _, bad := range cases {
|
||||
set := loadOne(t, "version: 1\njobs:\n x:\n command: convert\n"+
|
||||
" inputs:\n - path: \""+bad+"\"\n format: dbml\n"+
|
||||
" output:\n format: json\n path: o.json\n")
|
||||
if err := set.Validate(); err == nil {
|
||||
t.Fatalf("path %q: expected rejection", bad)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateOutputTraversalRejected(t *testing.T) {
|
||||
set := loadOne(t, "version: 1\njobs:\n x:\n command: convert\n"+
|
||||
" inputs:\n - path: a.dbml\n format: dbml\n"+
|
||||
" output:\n format: json\n path: ../../evil.json\n")
|
||||
if err := set.Validate(); err == nil {
|
||||
t.Fatal("expected output path traversal rejection")
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateConnEnvMustBeName(t *testing.T) {
|
||||
set := loadOne(t, "version: 1\njobs:\n x:\n command: convert\n"+
|
||||
" inputs:\n - format: pgsql\n conn_env: \"postgres://u:p@h/db\"\n"+
|
||||
" output:\n format: json\n path: o.json\n")
|
||||
if err := set.Validate(); err == nil || !strings.Contains(err.Error(), "environment variable name") {
|
||||
t.Fatalf("want conn_env name error, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateDependsOnUnknown(t *testing.T) {
|
||||
set := loadOne(t, "version: 1\njobs:\n x:\n command: convert\n depends_on: [nope]\n"+
|
||||
" inputs:\n - path: a.dbml\n format: dbml\n"+
|
||||
" output:\n format: json\n path: o.json\n")
|
||||
if err := set.Validate(); err == nil || !strings.Contains(err.Error(), "unknown job") {
|
||||
t.Fatalf("want unknown dependency error, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateDependencyCycle(t *testing.T) {
|
||||
content := "version: 1\njobs:\n" +
|
||||
jobBlock("a", "b") + jobBlock("b", "c") + jobBlock("c", "a")
|
||||
set := loadOne(t, content)
|
||||
err := set.Validate()
|
||||
if err == nil || !strings.Contains(err.Error(), "cycle") {
|
||||
t.Fatalf("want cycle error, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func jobBlock(name, dep string) string {
|
||||
return " " + name + ":\n command: convert\n depends_on: [" + dep + "]\n" +
|
||||
" inputs:\n - path: a.dbml\n format: dbml\n" +
|
||||
" output:\n format: json\n path: " + name + ".json\n"
|
||||
}
|
||||
|
||||
func TestPlanTopologicalOrder(t *testing.T) {
|
||||
content := "version: 1\njobs:\n" +
|
||||
" base:\n command: convert\n inputs:\n - path: a.dbml\n format: dbml\n output:\n format: json\n path: base.json\n" +
|
||||
" mid:\n command: convert\n depends_on: [base]\n inputs:\n - path: a.dbml\n format: dbml\n output:\n format: json\n path: mid.json\n" +
|
||||
" top:\n command: convert\n depends_on: [mid]\n inputs:\n - path: a.dbml\n format: dbml\n output:\n format: json\n path: top.json\n"
|
||||
set := loadOne(t, content)
|
||||
if err := set.Validate(); err != nil {
|
||||
t.Fatalf("validate: %v", err)
|
||||
}
|
||||
plan, err := set.Plan("top", true)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var order []string
|
||||
for _, j := range plan {
|
||||
order = append(order, j.Name)
|
||||
}
|
||||
if strings.Join(order, ",") != "base,mid,top" {
|
||||
t.Fatalf("plan order = %v, want [base mid top]", order)
|
||||
}
|
||||
|
||||
solo, err := set.Plan("top", false)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(solo) != 1 || solo[0].Name != "top" {
|
||||
t.Fatalf("no-deps plan = %v, want [top]", solo)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSafeJoinStaysInsideRoot(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
if _, err := SafeJoin(root, "sub/dir/file.sql"); err != nil {
|
||||
t.Fatalf("expected ok, got %v", err)
|
||||
}
|
||||
if _, err := SafeJoin(root, "../escape"); err == nil {
|
||||
t.Fatal("expected escape rejection")
|
||||
}
|
||||
if _, err := SafeJoin(root, "/abs"); err == nil {
|
||||
t.Fatal("expected absolute rejection")
|
||||
}
|
||||
}
|
||||
|
||||
func TestShippedExampleIsValid(t *testing.T) {
|
||||
path := filepath.Join("..", "..", "examples", "jobs", "relspec.yml")
|
||||
set, err := Load([]string{path})
|
||||
if err != nil {
|
||||
t.Fatalf("load example: %v", err)
|
||||
}
|
||||
if err := set.Validate(); err != nil {
|
||||
t.Fatalf("example manifest failed validation: %v", err)
|
||||
}
|
||||
if _, err := set.Plan("build-json", true); err != nil {
|
||||
t.Fatalf("plan example: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestScriptsListValidation(t *testing.T) {
|
||||
set := loadOne(t, "version: 1\njobs:\n s:\n command: scripts-list\n")
|
||||
if err := set.Validate(); err == nil || !strings.Contains(err.Error(), "script_dir") {
|
||||
t.Fatalf("want script_dir required error, got %v", err)
|
||||
}
|
||||
set = loadOne(t, "version: 1\njobs:\n s:\n command: scripts-list\n script_dirs: [migrations, extra]\n")
|
||||
if err := set.Validate(); err != nil {
|
||||
t.Fatalf("expected valid scripts-list job, got %v", err)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user