relspecgo/pkg/jobs/jobs.go

// Package jobs implements RelSpec declarative job files.
//
// A job file is a small YAML manifest that names one or more jobs and,
// for each job, the RelSpec command to run plus its inputs, output and
// options. It lets users run "relspec job run build-schema" instead of
// repeating long command lines.
//
// The job-file system is deliberately NOT a shell: "command" is a closed
// enum of vetted RelSpec workflows, every path is resolved relative to the
// directory holding the job file and may not escape it, and remote database
// credentials are referenced by environment-variable name only - never
// embedded in the manifest. All discovery, parsing and validation in this
// package is side-effect free; nothing here reads input schemas, opens
// database connections or writes output. Execution lives in the CLI layer
// and only runs after Validate and the caller's pre-flight checks pass.
package jobs

import (
	"fmt"
	"os"
	"path/filepath"
	"sort"
	"strings"

	"gopkg.in/yaml.v3"
)

// SchemaVersion is the only job-file schema version this build understands.
const SchemaVersion = 1

// Command names are a closed allow-list. Arbitrary strings are rejected.
const (
	CommandConvert     = "convert"      // read one or more schema files, optionally merge, write one output
	CommandMerge       = "merge"        // additive merge of two or more schema files into one output
	CommandScriptsList = "scripts-list" // deterministically list SQL scripts across one or more directories
)

// SupportedCommands lists every accepted command, in help order.
var SupportedCommands = []string{CommandConvert, CommandMerge, CommandScriptsList}

// readerFormats are the file-based input formats a job may declare (path).
var readerFormats = map[string]bool{
	"dbml": true, "dctx": true, "drawdb": true, "graphql": true, "json": true,
	"yaml": true, "gorm": true, "bun": true, "drizzle": true, "prisma": true,
	"typeorm": true, "sqlite": true,
}

// inputDBFormats are input formats that can only come from a live connection,
// referenced by conn_env.
var inputDBFormats = map[string]bool{"pgsql": true, "mssql": true}

// writerFormats are the output formats a job may declare.
var writerFormats = map[string]bool{
	"dbml": true, "dctx": true, "drawdb": true, "graphql": true, "json": true,
	"yaml": true, "gorm": true, "bun": true, "drizzle": true, "prisma": true,
	"typeorm": true, "pgsql": true, "mssql": true, "sqlite": true,
}

// execOutputFormats are output formats for which conn_env (execute against a
// live database) is supported instead of writing a file.
var execOutputFormats = map[string]bool{"pgsql": true}

// File is the on-disk shape of a single job file.
type File struct {
	Version int             `yaml:"version"`
	Jobs    map[string]*Job `yaml:"jobs"`
}

// Job is one named job within a job file.
type Job struct {
	// Name and SourceFile are populated by Load, not parsed from YAML.
	Name       string `yaml:"-"`
	SourceFile string `yaml:"-"`

	Command     string   `yaml:"command"`
	Description string   `yaml:"description"`
	DependsOn   []string `yaml:"depends_on"`
	Inputs      []Input  `yaml:"inputs"`
	ScriptDirs  []string `yaml:"script_dirs"`
	Output      *Output  `yaml:"output"`
	Options     Options  `yaml:"options"`
	Logfile     string   `yaml:"logfile"`
}

// Input is one declared input schema.
type Input struct {
	Path string `yaml:"path"`
	// Format is the RelSpec reader format (dbml, json, yaml, pgsql, ...).
	Format string `yaml:"format"`
	// ConnEnv is the NAME of an environment variable holding a connection
	// string, used with database formats. The value is never stored here.
	ConnEnv string `yaml:"conn_env"`
}

// Output is the declared output target.
type Output struct {
	Format    string `yaml:"format"`
	Path      string `yaml:"path"`
	ConnEnv   string `yaml:"conn_env"`
	Overwrite bool   `yaml:"overwrite"`
}

// Options carries the subset of command flags a job file may set.
type Options struct {
	FlattenSchema   bool   `yaml:"flatten_schema"`
	Schema          string `yaml:"schema"`
	Package         string `yaml:"package"`
	ContinueOnError bool   `yaml:"continue_on_error"`
	SkipRelations   bool   `yaml:"skip_relations"`
	SkipEnums       bool   `yaml:"skip_enums"`
	SkipViews       bool   `yaml:"skip_views"`
	SkipDomains     bool   `yaml:"skip_domains"`
	SkipSequences   bool   `yaml:"skip_sequences"`
}

// Dir returns the directory that a job's relative paths resolve against:
// the directory containing the job file that declared it.
func (j *Job) Dir() string { return filepath.Dir(j.SourceFile) }

// Set is the merged view of all discovered/selected job files.
type Set struct {
	// Files is the sorted list of job files that contributed jobs.
	Files []string
	// Jobs is keyed by job name.
	Jobs map[string]*Job
}

// Names returns all job names in deterministic (sorted) order.
func (s *Set) Names() []string {
	names := make([]string, 0, len(s.Jobs))
	for n := range s.Jobs {
		names = append(names, n)
	}
	sort.Strings(names)
	return names
}

// Discover returns the job files in dir in deterministic order. The default
// file "relspec.yml"/"relspec.yaml" sorts first, followed by named files
// "relspec.<name>.yml"/"relspec.<name>.yaml" in lexical order.
func Discover(dir string) ([]string, error) {
	if dir == "" {
		dir = "."
	}
	entries, err := os.ReadDir(dir)
	if err != nil {
		return nil, fmt.Errorf("failed to read directory %q: %w", dir, err)
	}
	var defaults, named []string
	for _, e := range entries {
		if e.IsDir() {
			continue
		}
		name := e.Name()
		if !isJobFileName(name) {
			continue
		}
		full := filepath.Join(dir, name)
		if name == "relspec.yml" || name == "relspec.yaml" {
			defaults = append(defaults, full)
		} else {
			named = append(named, full)
		}
	}
	sort.Strings(defaults)
	sort.Strings(named)
	return append(defaults, named...), nil
}

func isJobFileName(name string) bool {
	for _, ext := range []string{".yml", ".yaml"} {
		if name == "relspec"+ext {
			return true
		}
		if strings.HasPrefix(name, "relspec.") && strings.HasSuffix(name, ext) {
			return true
		}
	}
	return false
}

// Load parses every path, rejects unknown fields and unsupported versions,
// and merges all jobs into one Set. A job name defined by more than one file
// is a hard error. Load performs structural checks only; call Validate for
// full semantic validation.
func Load(paths []string) (*Set, error) {
	if len(paths) == 0 {
		return nil, fmt.Errorf("no job files found (looked for relspec.yml / relspec.<name>.yml)")
	}
	set := &Set{Jobs: map[string]*Job{}}
	origin := map[string]string{} // job name -> first file that defined it

	for _, path := range paths {
		data, err := os.ReadFile(path)
		if err != nil {
			return nil, fmt.Errorf("failed to read job file %q: %w", path, err)
		}
		dec := yaml.NewDecoder(strings.NewReader(string(data)))
		dec.KnownFields(true)
		var f File
		if err := dec.Decode(&f); err != nil {
			return nil, fmt.Errorf("invalid job file %q: %w", path, err)
		}
		if f.Version != SchemaVersion {
			return nil, fmt.Errorf("job file %q: unsupported version %d (expected %d)", path, f.Version, SchemaVersion)
		}
		if len(f.Jobs) == 0 {
			return nil, fmt.Errorf("job file %q: no jobs defined", path)
		}
		for name, job := range f.Jobs {
			if job == nil {
				return nil, fmt.Errorf("job file %q: job %q is empty", path, name)
			}
			if prev, dup := origin[name]; dup {
				return nil, fmt.Errorf("duplicate job %q defined in both %q and %q", name, prev, path)
			}
			job.Name = name
			job.SourceFile = path
			origin[name] = path
			set.Jobs[name] = job
		}
		set.Files = append(set.Files, path)
	}
	return set, nil
}

// Validate runs full semantic validation over the whole set and returns a
// single error describing every problem found. It never touches the
// filesystem beyond what Load already read; existence of input files and
// environment variables is checked by the caller immediately before
// execution.
func (s *Set) Validate() error {
	var errs []string
	for _, name := range s.Names() {
		for _, msg := range s.Jobs[name].validate() {
			errs = append(errs, fmt.Sprintf("job %q: %s", name, msg))
		}
	}
	// Dependency references + cycles.
	for _, name := range s.Names() {
		for _, dep := range s.Jobs[name].DependsOn {
			if _, ok := s.Jobs[dep]; !ok {
				errs = append(errs, fmt.Sprintf("job %q: depends_on unknown job %q", name, dep))
			}
		}
	}
	if cycle := s.findCycle(); cycle != "" {
		errs = append(errs, fmt.Sprintf("dependency cycle detected: %s", cycle))
	}
	if len(errs) > 0 {
		sort.Strings(errs)
		return fmt.Errorf("job file validation failed:\n  - %s", strings.Join(errs, "\n  - "))
	}
	return nil
}

func (j *Job) validate() []string {
	var e []string

	switch j.Command {
	case CommandConvert, CommandMerge, CommandScriptsList:
	case "":
		e = append(e, "missing command")
		return e
	default:
		e = append(e, fmt.Sprintf("unsupported command %q (supported: %s)", j.Command, strings.Join(SupportedCommands, ", ")))
		return e
	}

	// Path safety for every declared path.
	checkPath := func(label, p string) {
		if p == "" {
			return
		}
		if err := checkRelPath(p); err != nil {
			e = append(e, fmt.Sprintf("%s %q: %v", label, p, err))
		}
	}
	checkPath("logfile", j.Logfile)
	for _, in := range j.Inputs {
		checkPath("input path", in.Path)
	}
	for _, d := range j.ScriptDirs {
		checkPath("script_dir", d)
	}
	if j.Output != nil {
		checkPath("output path", j.Output.Path)
	}

	switch j.Command {
	case CommandConvert, CommandMerge:
		minInputs := 1
		if j.Command == CommandMerge {
			minInputs = 2
		}
		if len(j.Inputs) < minInputs {
			e = append(e, fmt.Sprintf("command %q requires at least %d input(s)", j.Command, minInputs))
		}
		for i, in := range j.Inputs {
			e = append(e, validateInput(i, in)...)
		}
		if len(j.ScriptDirs) > 0 {
			e = append(e, fmt.Sprintf("script_dirs is not valid for command %q", j.Command))
		}
		if j.Output == nil {
			e = append(e, "missing output")
		} else {
			e = append(e, validateOutput(*j.Output)...)
		}
	case CommandScriptsList:
		if len(j.ScriptDirs) == 0 {
			e = append(e, "command \"scripts-list\" requires at least one script_dir")
		}
		if len(j.Inputs) > 0 {
			e = append(e, "inputs is not valid for command \"scripts-list\"")
		}
		if j.Output != nil {
			e = append(e, "output is not valid for command \"scripts-list\"")
		}
	}
	return e
}

func validateInput(i int, in Input) []string {
	var e []string
	if in.Format == "" {
		e = append(e, fmt.Sprintf("input[%d]: missing format", i))
		return e
	}
	f := strings.ToLower(in.Format)
	switch {
	case inputDBFormats[f]:
		if in.ConnEnv == "" {
			e = append(e, fmt.Sprintf("input[%d]: format %q requires conn_env (an environment variable name)", i, in.Format))
		}
		if in.Path != "" {
			e = append(e, fmt.Sprintf("input[%d]: format %q takes conn_env, not path", i, in.Format))
		}
	case readerFormats[f]:
		if in.Path == "" {
			e = append(e, fmt.Sprintf("input[%d]: missing path", i))
		}
		if in.ConnEnv != "" {
			e = append(e, fmt.Sprintf("input[%d]: format %q does not use conn_env", i, in.Format))
		}
	default:
		e = append(e, fmt.Sprintf("input[%d]: unsupported input format %q", i, in.Format))
	}
	if looksLikeSecret(in.ConnEnv) {
		e = append(e, fmt.Sprintf("input[%d]: conn_env must be an environment variable name, not a connection string", i))
	}
	return e
}

func validateOutput(o Output) []string {
	var e []string
	if o.Format == "" {
		e = append(e, "output: missing format")
		return e
	}
	f := strings.ToLower(o.Format)
	if !writerFormats[f] {
		e = append(e, fmt.Sprintf("output: unsupported output format %q", o.Format))
		return e
	}
	if o.ConnEnv != "" {
		if !execOutputFormats[f] {
			e = append(e, fmt.Sprintf("output: conn_env (live database execution) is not supported for format %q", o.Format))
		}
		if o.Path != "" {
			e = append(e, "output: set either path or conn_env, not both")
		}
	} else if o.Path == "" {
		e = append(e, "output: missing path")
	}
	if looksLikeSecret(o.ConnEnv) {
		e = append(e, "output: conn_env must be an environment variable name, not a connection string")
	}
	return e
}

// looksLikeSecret reports whether s looks like a connection string rather
// than a bare environment-variable name.
func looksLikeSecret(s string) bool {
	if s == "" {
		return false
	}
	return strings.ContainsAny(s, ":/@ =") || strings.Contains(s, "//")
}

// checkRelPath rejects absolute paths and any path that escapes its root.
func checkRelPath(p string) error {
	if p == "" {
		return fmt.Errorf("empty path")
	}
	if filepath.IsAbs(p) {
		return fmt.Errorf("absolute paths are not allowed; use a path relative to the job file")
	}
	if strings.HasPrefix(p, "~") {
		return fmt.Errorf("home-relative paths are not allowed")
	}
	clean := filepath.ToSlash(filepath.Clean(p))
	if clean == ".." || strings.HasPrefix(clean, "../") {
		return fmt.Errorf("path escapes the job file directory")
	}
	return nil
}

// SafeJoin resolves rel against root and guarantees the result stays inside
// root. It is the single choke point for turning a manifest path into a
// filesystem path.
func SafeJoin(root, rel string) (string, error) {
	if err := checkRelPath(rel); err != nil {
		return "", err
	}
	absRoot, err := filepath.Abs(root)
	if err != nil {
		return "", err
	}
	joined := filepath.Join(absRoot, rel)
	rp, err := filepath.Rel(absRoot, joined)
	if err != nil {
		return "", err
	}
	if rp == ".." || strings.HasPrefix(rp, ".."+string(filepath.Separator)) {
		return "", fmt.Errorf("path %q escapes the job file directory", rel)
	}
	return joined, nil
}

// Plan returns the jobs to execute for name in dependency order. When
// includeDeps is false only the named job is returned (its declared
// dependencies are still validated to exist and be acyclic by Validate).
func (s *Set) Plan(name string, includeDeps bool) ([]*Job, error) {
	root, ok := s.Jobs[name]
	if !ok {
		return nil, fmt.Errorf("unknown job %q (known: %s)", name, strings.Join(s.Names(), ", "))
	}
	if !includeDeps {
		return []*Job{root}, nil
	}
	var order []*Job
	visited := map[string]bool{}
	inProgress := map[string]bool{}
	var visit func(n string) error
	visit = func(n string) error {
		if visited[n] {
			return nil
		}
		if inProgress[n] {
			return fmt.Errorf("dependency cycle at job %q", n)
		}
		inProgress[n] = true
		j := s.Jobs[n]
		deps := append([]string(nil), j.DependsOn...)
		sort.Strings(deps)
		for _, d := range deps {
			if _, ok := s.Jobs[d]; !ok {
				return fmt.Errorf("job %q depends on unknown job %q", n, d)
			}
			if err := visit(d); err != nil {
				return err
			}
		}
		inProgress[n] = false
		visited[n] = true
		order = append(order, j)
		return nil
	}
	if err := visit(name); err != nil {
		return nil, err
	}
	return order, nil
}

// findCycle returns a human-readable cycle path, or "" if the graph is acyclic.
func (s *Set) findCycle() string {
	color := map[string]int{} // 0 unvisited, 1 in progress, 2 done
	var stack []string
	var dfs func(n string) []string
	dfs = func(n string) []string {
		color[n] = 1
		stack = append(stack, n)
		deps := append([]string(nil), s.Jobs[n].DependsOn...)
		sort.Strings(deps)
		for _, d := range deps {
			if _, ok := s.Jobs[d]; !ok {
				continue
			}
			switch color[d] {
			case 0:
				if c := dfs(d); c != nil {
					return c
				}
			case 1:
				// Found a back edge; build the cycle slice.
				for i, x := range stack {
					if x == d {
						return append(append([]string(nil), stack[i:]...), d)
					}
				}
				return []string{d, d}
			}
		}
		stack = stack[:len(stack)-1]
		color[n] = 2
		return nil
	}
	for _, n := range s.Names() {
		if color[n] == 0 {
			if c := dfs(n); c != nil {
				return strings.Join(c, " -> ")
			}
		}
	}
	return ""
}