// Package jobs implements RelSpec declarative job files. // // A job file is a small YAML manifest that names one or more jobs and, // for each job, the RelSpec command to run plus its inputs, output and // options. It lets users run "relspec job run build-schema" instead of // repeating long command lines. // // The job-file system is deliberately NOT a shell: "command" is a closed // enum of vetted RelSpec workflows, every path is resolved relative to the // directory holding the job file and may not escape it, and remote database // credentials are referenced by environment-variable name only - never // embedded in the manifest. All discovery, parsing and validation in this // package is side-effect free; nothing here reads input schemas, opens // database connections or writes output. Execution lives in the CLI layer // and only runs after Validate and the caller's pre-flight checks pass. package jobs import ( "fmt" "os" "path/filepath" "sort" "strings" "gopkg.in/yaml.v3" ) // SchemaVersion is the only job-file schema version this build understands. const SchemaVersion = 1 // Command names are a closed allow-list. Arbitrary strings are rejected. const ( CommandConvert = "convert" // read one or more schema files, optionally merge, write one output CommandMerge = "merge" // additive merge of two or more schema files into one output CommandScriptsList = "scripts-list" // deterministically list SQL scripts across one or more directories ) // SupportedCommands lists every accepted command, in help order. var SupportedCommands = []string{CommandConvert, CommandMerge, CommandScriptsList} // readerFormats are the file-based input formats a job may declare (path). var readerFormats = map[string]bool{ "dbml": true, "dctx": true, "drawdb": true, "graphql": true, "json": true, "yaml": true, "gorm": true, "bun": true, "drizzle": true, "prisma": true, "typeorm": true, "sqlite": true, } // inputDBFormats are input formats that can only come from a live connection, // referenced by conn_env. var inputDBFormats = map[string]bool{"pgsql": true, "mssql": true} // writerFormats are the output formats a job may declare. var writerFormats = map[string]bool{ "dbml": true, "dctx": true, "drawdb": true, "graphql": true, "json": true, "yaml": true, "gorm": true, "bun": true, "drizzle": true, "prisma": true, "typeorm": true, "pgsql": true, "mssql": true, "sqlite": true, } // execOutputFormats are output formats for which conn_env (execute against a // live database) is supported instead of writing a file. var execOutputFormats = map[string]bool{"pgsql": true} // File is the on-disk shape of a single job file. type File struct { Version int `yaml:"version"` Jobs map[string]*Job `yaml:"jobs"` } // Job is one named job within a job file. type Job struct { // Name and SourceFile are populated by Load, not parsed from YAML. Name string `yaml:"-"` SourceFile string `yaml:"-"` Command string `yaml:"command"` Description string `yaml:"description"` DependsOn []string `yaml:"depends_on"` Inputs []Input `yaml:"inputs"` ScriptDirs []string `yaml:"script_dirs"` Output *Output `yaml:"output"` Options Options `yaml:"options"` Logfile string `yaml:"logfile"` } // Input is one declared input schema. type Input struct { Path string `yaml:"path"` // Format is the RelSpec reader format (dbml, json, yaml, pgsql, ...). Format string `yaml:"format"` // ConnEnv is the NAME of an environment variable holding a connection // string, used with database formats. The value is never stored here. ConnEnv string `yaml:"conn_env"` } // Output is the declared output target. type Output struct { Format string `yaml:"format"` Path string `yaml:"path"` ConnEnv string `yaml:"conn_env"` Overwrite bool `yaml:"overwrite"` } // Options carries the subset of command flags a job file may set. type Options struct { FlattenSchema bool `yaml:"flatten_schema"` Schema string `yaml:"schema"` Package string `yaml:"package"` ContinueOnError bool `yaml:"continue_on_error"` SkipRelations bool `yaml:"skip_relations"` SkipEnums bool `yaml:"skip_enums"` SkipViews bool `yaml:"skip_views"` SkipDomains bool `yaml:"skip_domains"` SkipSequences bool `yaml:"skip_sequences"` } // Dir returns the directory that a job's relative paths resolve against: // the directory containing the job file that declared it. func (j *Job) Dir() string { return filepath.Dir(j.SourceFile) } // Set is the merged view of all discovered/selected job files. type Set struct { // Files is the sorted list of job files that contributed jobs. Files []string // Jobs is keyed by job name. Jobs map[string]*Job } // Names returns all job names in deterministic (sorted) order. func (s *Set) Names() []string { names := make([]string, 0, len(s.Jobs)) for n := range s.Jobs { names = append(names, n) } sort.Strings(names) return names } // Discover returns the job files in dir in deterministic order. The default // file "relspec.yml"/"relspec.yaml" sorts first, followed by named files // "relspec..yml"/"relspec..yaml" in lexical order. func Discover(dir string) ([]string, error) { if dir == "" { dir = "." } entries, err := os.ReadDir(dir) if err != nil { return nil, fmt.Errorf("failed to read directory %q: %w", dir, err) } var defaults, named []string for _, e := range entries { if e.IsDir() { continue } name := e.Name() if !isJobFileName(name) { continue } full := filepath.Join(dir, name) if name == "relspec.yml" || name == "relspec.yaml" { defaults = append(defaults, full) } else { named = append(named, full) } } sort.Strings(defaults) sort.Strings(named) return append(defaults, named...), nil } func isJobFileName(name string) bool { for _, ext := range []string{".yml", ".yaml"} { if name == "relspec"+ext { return true } if strings.HasPrefix(name, "relspec.") && strings.HasSuffix(name, ext) { return true } } return false } // Load parses every path, rejects unknown fields and unsupported versions, // and merges all jobs into one Set. A job name defined by more than one file // is a hard error. Load performs structural checks only; call Validate for // full semantic validation. func Load(paths []string) (*Set, error) { if len(paths) == 0 { return nil, fmt.Errorf("no job files found (looked for relspec.yml / relspec..yml)") } set := &Set{Jobs: map[string]*Job{}} origin := map[string]string{} // job name -> first file that defined it for _, path := range paths { data, err := os.ReadFile(path) if err != nil { return nil, fmt.Errorf("failed to read job file %q: %w", path, err) } dec := yaml.NewDecoder(strings.NewReader(string(data))) dec.KnownFields(true) var f File if err := dec.Decode(&f); err != nil { return nil, fmt.Errorf("invalid job file %q: %w", path, err) } if f.Version != SchemaVersion { return nil, fmt.Errorf("job file %q: unsupported version %d (expected %d)", path, f.Version, SchemaVersion) } if len(f.Jobs) == 0 { return nil, fmt.Errorf("job file %q: no jobs defined", path) } for name, job := range f.Jobs { if job == nil { return nil, fmt.Errorf("job file %q: job %q is empty", path, name) } if prev, dup := origin[name]; dup { return nil, fmt.Errorf("duplicate job %q defined in both %q and %q", name, prev, path) } job.Name = name job.SourceFile = path origin[name] = path set.Jobs[name] = job } set.Files = append(set.Files, path) } return set, nil } // Validate runs full semantic validation over the whole set and returns a // single error describing every problem found. It never touches the // filesystem beyond what Load already read; existence of input files and // environment variables is checked by the caller immediately before // execution. func (s *Set) Validate() error { var errs []string for _, name := range s.Names() { for _, msg := range s.Jobs[name].validate() { errs = append(errs, fmt.Sprintf("job %q: %s", name, msg)) } } // Dependency references + cycles. for _, name := range s.Names() { for _, dep := range s.Jobs[name].DependsOn { if _, ok := s.Jobs[dep]; !ok { errs = append(errs, fmt.Sprintf("job %q: depends_on unknown job %q", name, dep)) } } } if cycle := s.findCycle(); cycle != "" { errs = append(errs, fmt.Sprintf("dependency cycle detected: %s", cycle)) } if len(errs) > 0 { sort.Strings(errs) return fmt.Errorf("job file validation failed:\n - %s", strings.Join(errs, "\n - ")) } return nil } func (j *Job) validate() []string { var e []string switch j.Command { case CommandConvert, CommandMerge, CommandScriptsList: case "": e = append(e, "missing command") return e default: e = append(e, fmt.Sprintf("unsupported command %q (supported: %s)", j.Command, strings.Join(SupportedCommands, ", "))) return e } // Path safety for every declared path. checkPath := func(label, p string) { if p == "" { return } if err := checkRelPath(p); err != nil { e = append(e, fmt.Sprintf("%s %q: %v", label, p, err)) } } checkPath("logfile", j.Logfile) for _, in := range j.Inputs { checkPath("input path", in.Path) } for _, d := range j.ScriptDirs { checkPath("script_dir", d) } if j.Output != nil { checkPath("output path", j.Output.Path) } switch j.Command { case CommandConvert, CommandMerge: minInputs := 1 if j.Command == CommandMerge { minInputs = 2 } if len(j.Inputs) < minInputs { e = append(e, fmt.Sprintf("command %q requires at least %d input(s)", j.Command, minInputs)) } for i, in := range j.Inputs { e = append(e, validateInput(i, in)...) } if len(j.ScriptDirs) > 0 { e = append(e, fmt.Sprintf("script_dirs is not valid for command %q", j.Command)) } if j.Output == nil { e = append(e, "missing output") } else { e = append(e, validateOutput(*j.Output)...) } case CommandScriptsList: if len(j.ScriptDirs) == 0 { e = append(e, "command \"scripts-list\" requires at least one script_dir") } if len(j.Inputs) > 0 { e = append(e, "inputs is not valid for command \"scripts-list\"") } if j.Output != nil { e = append(e, "output is not valid for command \"scripts-list\"") } } return e } func validateInput(i int, in Input) []string { var e []string if in.Format == "" { e = append(e, fmt.Sprintf("input[%d]: missing format", i)) return e } f := strings.ToLower(in.Format) switch { case inputDBFormats[f]: if in.ConnEnv == "" { e = append(e, fmt.Sprintf("input[%d]: format %q requires conn_env (an environment variable name)", i, in.Format)) } if in.Path != "" { e = append(e, fmt.Sprintf("input[%d]: format %q takes conn_env, not path", i, in.Format)) } case readerFormats[f]: if in.Path == "" { e = append(e, fmt.Sprintf("input[%d]: missing path", i)) } if in.ConnEnv != "" { e = append(e, fmt.Sprintf("input[%d]: format %q does not use conn_env", i, in.Format)) } default: e = append(e, fmt.Sprintf("input[%d]: unsupported input format %q", i, in.Format)) } if looksLikeSecret(in.ConnEnv) { e = append(e, fmt.Sprintf("input[%d]: conn_env must be an environment variable name, not a connection string", i)) } return e } func validateOutput(o Output) []string { var e []string if o.Format == "" { e = append(e, "output: missing format") return e } f := strings.ToLower(o.Format) if !writerFormats[f] { e = append(e, fmt.Sprintf("output: unsupported output format %q", o.Format)) return e } if o.ConnEnv != "" { if !execOutputFormats[f] { e = append(e, fmt.Sprintf("output: conn_env (live database execution) is not supported for format %q", o.Format)) } if o.Path != "" { e = append(e, "output: set either path or conn_env, not both") } } else if o.Path == "" { e = append(e, "output: missing path") } if looksLikeSecret(o.ConnEnv) { e = append(e, "output: conn_env must be an environment variable name, not a connection string") } return e } // looksLikeSecret reports whether s looks like a connection string rather // than a bare environment-variable name. func looksLikeSecret(s string) bool { if s == "" { return false } return strings.ContainsAny(s, ":/@ =") || strings.Contains(s, "//") } // checkRelPath rejects absolute paths and any path that escapes its root. func checkRelPath(p string) error { if p == "" { return fmt.Errorf("empty path") } if filepath.IsAbs(p) { return fmt.Errorf("absolute paths are not allowed; use a path relative to the job file") } if strings.HasPrefix(p, "~") { return fmt.Errorf("home-relative paths are not allowed") } clean := filepath.ToSlash(filepath.Clean(p)) if clean == ".." || strings.HasPrefix(clean, "../") { return fmt.Errorf("path escapes the job file directory") } return nil } // SafeJoin resolves rel against root and guarantees the result stays inside // root. It is the single choke point for turning a manifest path into a // filesystem path. func SafeJoin(root, rel string) (string, error) { if err := checkRelPath(rel); err != nil { return "", err } absRoot, err := filepath.Abs(root) if err != nil { return "", err } joined := filepath.Join(absRoot, rel) rp, err := filepath.Rel(absRoot, joined) if err != nil { return "", err } if rp == ".." || strings.HasPrefix(rp, ".."+string(filepath.Separator)) { return "", fmt.Errorf("path %q escapes the job file directory", rel) } return joined, nil } // Plan returns the jobs to execute for name in dependency order. When // includeDeps is false only the named job is returned (its declared // dependencies are still validated to exist and be acyclic by Validate). func (s *Set) Plan(name string, includeDeps bool) ([]*Job, error) { root, ok := s.Jobs[name] if !ok { return nil, fmt.Errorf("unknown job %q (known: %s)", name, strings.Join(s.Names(), ", ")) } if !includeDeps { return []*Job{root}, nil } var order []*Job visited := map[string]bool{} inProgress := map[string]bool{} var visit func(n string) error visit = func(n string) error { if visited[n] { return nil } if inProgress[n] { return fmt.Errorf("dependency cycle at job %q", n) } inProgress[n] = true j := s.Jobs[n] deps := append([]string(nil), j.DependsOn...) sort.Strings(deps) for _, d := range deps { if _, ok := s.Jobs[d]; !ok { return fmt.Errorf("job %q depends on unknown job %q", n, d) } if err := visit(d); err != nil { return err } } inProgress[n] = false visited[n] = true order = append(order, j) return nil } if err := visit(name); err != nil { return nil, err } return order, nil } // findCycle returns a human-readable cycle path, or "" if the graph is acyclic. func (s *Set) findCycle() string { color := map[string]int{} // 0 unvisited, 1 in progress, 2 done var stack []string var dfs func(n string) []string dfs = func(n string) []string { color[n] = 1 stack = append(stack, n) deps := append([]string(nil), s.Jobs[n].DependsOn...) sort.Strings(deps) for _, d := range deps { if _, ok := s.Jobs[d]; !ok { continue } switch color[d] { case 0: if c := dfs(d); c != nil { return c } case 1: // Found a back edge; build the cycle slice. for i, x := range stack { if x == d { return append(append([]string(nil), stack[i:]...), d) } } return []string{d, d} } } stack = stack[:len(stack)-1] color[n] = 2 return nil } for _, n := range s.Names() { if color[n] == 0 { if c := dfs(n); c != nil { return strings.Join(c, " -> ") } } } return "" }