feat(job): complete deferred job-file features
Implements the remaining items from issue #20: - version is now forward-permissive: any value >= 1 is accepted; a newer-than-known version loads best-effort (unknown fields ignored, warning printed) instead of hard-failing on "must be 1" - from_job input reference: `inputs: [{ from_job: <job> }]` resolves to that job's single-file output + format and implies a dependency edge; combined depends_on + from_job graph gets topological ordering and cycle detection - logfile size-rotation, on by default (5MB, keep 3), overridable per job (log_max_size / log_keep) or file-wide via a top-level defaults block - new commands: split (schema/table subsetting via select:), inspect (rule validation -> markdown/json report, fails job on enforced-rule errors), diff (compare exactly two schemas, never fails), scripts-exec (run SQL script dirs against a live PostgreSQL database) - atomic single-file output/report writes (temp file + rename) - symlink-escape hardening in SafeJoin via EvalSymlinks preflight Updates docs/JOB_FILES.md and examples/jobs/relspec.yml accordingly.
This commit is contained in:
Hein
@@ -376,6 +376,286 @@ jobs:
|
||||
}
|
||||
}
|
||||
|
||||
func TestJobRun_SplitJob(t *testing.T) {
|
||||
dir := jobFixture(t, `version: 1
|
||||
jobs:
|
||||
extract:
|
||||
command: split
|
||||
inputs:
|
||||
- path: schema/core.dbml
|
||||
format: dbml
|
||||
- path: schema/tenant.dbml
|
||||
format: dbml
|
||||
select:
|
||||
tables: [users]
|
||||
output:
|
||||
format: json
|
||||
path: build/subset.json
|
||||
overwrite: true
|
||||
`)
|
||||
set := mustLoadSet(t, filepath.Join(dir, "relspec.yml"))
|
||||
if err := executeJobPlan(set, "extract", false, false, &bytes.Buffer{}); err != nil {
|
||||
t.Fatalf("execute split job: %v", err)
|
||||
}
|
||||
out, err := os.ReadFile(filepath.Join(dir, "build", "subset.json"))
|
||||
if err != nil {
|
||||
t.Fatalf("read split output: %v", err)
|
||||
}
|
||||
s := string(out)
|
||||
if !strings.Contains(s, "users") {
|
||||
t.Fatalf("split output missing selected table:\n%s", s)
|
||||
}
|
||||
if strings.Contains(s, "posts") {
|
||||
t.Fatalf("split output should have excluded posts:\n%s", s)
|
||||
}
|
||||
}
|
||||
|
||||
func TestJobRun_InspectJob(t *testing.T) {
|
||||
dir := jobFixture(t, `version: 1
|
||||
jobs:
|
||||
lint:
|
||||
command: inspect
|
||||
inputs:
|
||||
- path: schema/core.dbml
|
||||
format: dbml
|
||||
report:
|
||||
format: json
|
||||
path: build/report.json
|
||||
overwrite: true
|
||||
logfile: .relspec/lint.log
|
||||
`)
|
||||
set := mustLoadSet(t, filepath.Join(dir, "relspec.yml"))
|
||||
// Default rules only warn, so the job succeeds.
|
||||
if err := executeJobPlan(set, "lint", false, false, &bytes.Buffer{}); err != nil {
|
||||
t.Fatalf("execute inspect job: %v", err)
|
||||
}
|
||||
if _, err := os.ReadFile(filepath.Join(dir, "build", "report.json")); err != nil {
|
||||
t.Fatalf("expected report file: %v", err)
|
||||
}
|
||||
logData, _ := os.ReadFile(filepath.Join(dir, ".relspec", "lint.log"))
|
||||
if !strings.Contains(string(logData), "inspect:") {
|
||||
t.Fatalf("logfile missing inspect summary:\n%s", logData)
|
||||
}
|
||||
}
|
||||
|
||||
func TestJobRun_InspectJobFailsOnRuleError(t *testing.T) {
|
||||
dir := jobFixture(t, `version: 1
|
||||
jobs:
|
||||
lint:
|
||||
command: inspect
|
||||
inputs:
|
||||
- path: schema/core.dbml
|
||||
format: dbml
|
||||
rules: rules.yaml
|
||||
report:
|
||||
format: json
|
||||
path: build/report.json
|
||||
overwrite: true
|
||||
logfile: .relspec/lint.log
|
||||
`)
|
||||
// A rule set to "error" level for a violation the fixture triggers.
|
||||
writeFile(t, filepath.Join(dir, "rules.yaml"), `version: "1.0"
|
||||
rules:
|
||||
primary_key_naming:
|
||||
enabled: enforce
|
||||
function: primary_key_naming
|
||||
pattern: "^id_"
|
||||
message: "Primary key columns should start with id_"
|
||||
`)
|
||||
set := mustLoadSet(t, filepath.Join(dir, "relspec.yml"))
|
||||
err := executeJobPlan(set, "lint", false, false, &bytes.Buffer{})
|
||||
if err == nil || !strings.Contains(err.Error(), "error(s)") {
|
||||
t.Fatalf("expected inspect job to fail on rule error, got %v", err)
|
||||
}
|
||||
logData, _ := os.ReadFile(filepath.Join(dir, ".relspec", "lint.log"))
|
||||
if !strings.Contains(string(logData), "FAILED") {
|
||||
t.Fatalf("failed inspect job should log FAILED:\n%s", logData)
|
||||
}
|
||||
}
|
||||
|
||||
func TestJobRun_DiffJob(t *testing.T) {
|
||||
dir := jobFixture(t, `version: 1
|
||||
jobs:
|
||||
compare:
|
||||
command: diff
|
||||
inputs:
|
||||
- path: schema/core.dbml
|
||||
format: dbml
|
||||
- path: schema/tenant.dbml
|
||||
format: dbml
|
||||
report:
|
||||
format: json
|
||||
path: build/diff.json
|
||||
overwrite: true
|
||||
`)
|
||||
set := mustLoadSet(t, filepath.Join(dir, "relspec.yml"))
|
||||
if err := executeJobPlan(set, "compare", false, false, &bytes.Buffer{}); err != nil {
|
||||
t.Fatalf("execute diff job: %v", err)
|
||||
}
|
||||
out, err := os.ReadFile(filepath.Join(dir, "build", "diff.json"))
|
||||
if err != nil {
|
||||
t.Fatalf("read diff report: %v", err)
|
||||
}
|
||||
if len(out) == 0 {
|
||||
t.Fatal("diff report is empty")
|
||||
}
|
||||
}
|
||||
|
||||
func TestJobRun_FromJobWiring(t *testing.T) {
|
||||
dir := jobFixture(t, `version: 1
|
||||
jobs:
|
||||
a:
|
||||
command: convert
|
||||
inputs:
|
||||
- path: schema/core.dbml
|
||||
format: dbml
|
||||
output:
|
||||
format: json
|
||||
path: build/a.json
|
||||
overwrite: true
|
||||
b:
|
||||
command: convert
|
||||
inputs:
|
||||
- from_job: a
|
||||
output:
|
||||
format: yaml
|
||||
path: build/b.yaml
|
||||
overwrite: true
|
||||
`)
|
||||
set := mustLoadSet(t, filepath.Join(dir, "relspec.yml"))
|
||||
if err := executeJobPlan(set, "b", false, false, &bytes.Buffer{}); err != nil {
|
||||
t.Fatalf("execute from_job chain: %v", err)
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(dir, "build", "a.json")); err != nil {
|
||||
t.Fatalf("producer output missing: %v", err)
|
||||
}
|
||||
out, err := os.ReadFile(filepath.Join(dir, "build", "b.yaml"))
|
||||
if err != nil {
|
||||
t.Fatalf("consumer output missing: %v", err)
|
||||
}
|
||||
if !strings.Contains(string(out), "users") {
|
||||
t.Fatalf("consumer did not consume producer output:\n%s", out)
|
||||
}
|
||||
}
|
||||
|
||||
func TestJobRun_LogRotation(t *testing.T) {
|
||||
dir := jobFixture(t, `version: 1
|
||||
jobs:
|
||||
build:
|
||||
command: scripts-list
|
||||
script_dirs: [migrations]
|
||||
log_max_size: "150B"
|
||||
log_keep: 2
|
||||
logfile: .relspec/build.log
|
||||
`)
|
||||
writeFile(t, filepath.Join(dir, "migrations", "1_001_a.sql"), "CREATE TABLE a();\n")
|
||||
logPath := filepath.Join(dir, ".relspec", "build.log")
|
||||
writeFile(t, logPath, strings.Repeat("x", 300)+"\n")
|
||||
|
||||
set := mustLoadSet(t, filepath.Join(dir, "relspec.yml"))
|
||||
if err := executeJobPlan(set, "build", false, false, &bytes.Buffer{}); err != nil {
|
||||
t.Fatalf("execute job: %v", err)
|
||||
}
|
||||
rotated, err := os.ReadFile(logPath + ".1")
|
||||
if err != nil {
|
||||
t.Fatalf("expected rotated logfile build.log.1: %v", err)
|
||||
}
|
||||
if !strings.Contains(string(rotated), strings.Repeat("x", 300)) {
|
||||
t.Fatalf("rotated logfile should hold the old content")
|
||||
}
|
||||
fresh, err := os.ReadFile(logPath)
|
||||
if err != nil {
|
||||
t.Fatalf("expected fresh logfile: %v", err)
|
||||
}
|
||||
if strings.Contains(string(fresh), strings.Repeat("x", 300)) {
|
||||
t.Fatalf("fresh logfile should not contain the rotated-out content:\n%s", fresh)
|
||||
}
|
||||
if !strings.Contains(string(fresh), "OK") {
|
||||
t.Fatalf("fresh logfile should hold the new run:\n%s", fresh)
|
||||
}
|
||||
}
|
||||
|
||||
func TestJobRun_AtomicOutputLeavesOriginalOnFailure(t *testing.T) {
|
||||
dir := jobFixture(t, `version: 1
|
||||
jobs:
|
||||
x:
|
||||
command: convert
|
||||
inputs:
|
||||
- path: schema/core.dbml
|
||||
format: dbml
|
||||
output:
|
||||
format: json
|
||||
path: build/out.json
|
||||
overwrite: true
|
||||
`)
|
||||
// Seed the destination, then make its parent directory read-only so the
|
||||
// rename step fails. The seeded file must survive intact.
|
||||
seeded := filepath.Join(dir, "build", "out.json")
|
||||
writeFile(t, seeded, `{"seeded":true}`)
|
||||
if err := os.Chmod(filepath.Join(dir, "build"), 0o500); err != nil {
|
||||
t.Skipf("cannot chmod: %v", err)
|
||||
}
|
||||
t.Cleanup(func() { _ = os.Chmod(filepath.Join(dir, "build"), 0o755) })
|
||||
|
||||
set := mustLoadSet(t, filepath.Join(dir, "relspec.yml"))
|
||||
if err := executeJobPlan(set, "x", false, false, &bytes.Buffer{}); err == nil {
|
||||
t.Skip("write unexpectedly succeeded (running as root?)")
|
||||
}
|
||||
if err := os.Chmod(filepath.Join(dir, "build"), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
data, err := os.ReadFile(seeded)
|
||||
if err != nil {
|
||||
t.Fatalf("seeded file gone: %v", err)
|
||||
}
|
||||
if !strings.Contains(string(data), "seeded") {
|
||||
t.Fatalf("seeded file was corrupted: %s", data)
|
||||
}
|
||||
}
|
||||
|
||||
func TestJobRun_ScriptsExecMissingConnEnv(t *testing.T) {
|
||||
dir := jobFixture(t, `version: 1
|
||||
jobs:
|
||||
migrate:
|
||||
command: scripts-exec
|
||||
script_dirs: [migrations]
|
||||
output:
|
||||
conn_env: RELSPEC_TEST_EXEC_MISSING
|
||||
logfile: .relspec/migrate.log
|
||||
`)
|
||||
writeFile(t, filepath.Join(dir, "migrations", "1_001_a.sql"), "CREATE TABLE a();\n")
|
||||
os.Unsetenv("RELSPEC_TEST_EXEC_MISSING")
|
||||
set := mustLoadSet(t, filepath.Join(dir, "relspec.yml"))
|
||||
err := executeJobPlan(set, "migrate", false, false, &bytes.Buffer{})
|
||||
if err == nil || !strings.Contains(err.Error(), "conn_env") {
|
||||
t.Fatalf("expected missing conn_env error, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestJobRun_ScriptsExecDryRun(t *testing.T) {
|
||||
dir := jobFixture(t, `version: 1
|
||||
jobs:
|
||||
migrate:
|
||||
command: scripts-exec
|
||||
script_dirs: [migrations]
|
||||
output:
|
||||
conn_env: RELSPEC_TEST_EXEC_CONN
|
||||
`)
|
||||
writeFile(t, filepath.Join(dir, "migrations", "1_001_a.sql"), "CREATE TABLE a();\n")
|
||||
t.Setenv("RELSPEC_TEST_EXEC_CONN", "postgres://u:secretpw@h/db")
|
||||
set := mustLoadSet(t, filepath.Join(dir, "relspec.yml"))
|
||||
var buf bytes.Buffer
|
||||
if err := executeJobPlan(set, "migrate", true, false, &buf); err != nil {
|
||||
t.Fatalf("dry run: %v", err)
|
||||
}
|
||||
if strings.Contains(buf.String(), "secretpw") {
|
||||
t.Fatalf("plan leaked secret:\n%s", buf.String())
|
||||
}
|
||||
if !strings.Contains(buf.String(), "env:RELSPEC_TEST_EXEC_CONN") {
|
||||
t.Fatalf("plan should name the env var:\n%s", buf.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestJobRun_TemplDatabaseMode(t *testing.T) {
|
||||
dir := jobFixture(t, `version: 1
|
||||
jobs:
|
||||
|
||||
Reference in New Issue
Block a user