Merge pull request 'feat(cli): add batch command for converting multiple inputs (#38)' (#49) from issue-38-batch-processing into master

Reviewed-on: #49
2026-10-03 18:14:30 +00:00
parent 720476fd6e 47108809aa
commit 66b90ca54b
4 changed files with 369 additions and 0 deletions
@@ -53,6 +53,17 @@ connections or `--dry-run`). Source files are polled every `--watch-interval`
 is ignored so generating into the source tree does not loop. Conversion errors
 are printed and watching continues.

+### `batch` — Convert many inputs in one run
+
+Converts each input independently (one output per input, unlike `--from-list`
+which merges). `--input` takes paths or globs; outputs go to `--to-dir`.
+Use `--keep-going` to continue past failures (exit is still non-zero) and
+`--dry-run` to validate without writing. For named workflows see `relspec job run`.
+
+```bash
+relspec batch --from dbml --input "schemas/*.dbml" --to json --to-dir out/
+```
+
 PostgreSQL connections opened by relspec set `application_name` by default to
 `relspecgo/<version>` (with component suffixes internally, e.g. readers/writers).
 If you need a custom value, provide `application_name` explicitly in the connection
@@ -0,0 +1,214 @@
+package main
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"sort"
+	"strings"
+
+	"github.com/spf13/cobra"
+)
+
+var (
+	batchSourceType      string
+	batchInputs          []string
+	batchTargetType      string
+	batchTargetDir       string
+	batchPackageName     string
+	batchSchemaFilter    string
+	batchFlattenSchema   bool
+	batchNullableTypes   string
+	batchNullableArrays  string
+	batchContinueOnError bool
+	batchKeepGoing       bool
+	batchDryRun          bool
+)
+
+var batchCmd = &cobra.Command{
+	Use:   "batch",
+	Short: "Convert many input files to a target format in one run",
+	Long: `Convert each input file independently to the target format.
+
+Unlike 'convert --from-list', which merges all inputs into one output, batch
+mode writes one output per input into --to-dir. The output is named after the
+input file (without its extension). Directory-style targets (gorm, bun,
+drizzle) get a sub-directory per input.
+
+Inputs are given with --input, which accepts file paths and glob patterns and
+may be repeated or comma-separated. Inputs are processed in sorted order and
+duplicates are removed. The command exits non-zero if any input fails.
+
+For named, multi-step workflows use 'relspec job run' instead.
+
+Examples:
+  # Convert every DBML file in a directory to JSON
+  relspec batch --from dbml --input "schemas/*.dbml" --to json --to-dir out/
+
+  # Convert specific files to GORM models, one package directory per input
+  relspec batch --from json --input a.json,b.json \
+                --to gorm --to-dir models/ --package models
+
+  # Validate everything first, writing nothing
+  relspec batch --from yaml --input "specs/*.yaml" --to pgsql --to-dir sql/ --dry-run
+
+  # Report all failures instead of stopping at the first
+  relspec batch --from json --input "*.json" --to yaml --to-dir out/ --keep-going`,
+	RunE: runBatch,
+}
+
+func init() {
+	batchCmd.Flags().StringVar(&batchSourceType, "from", "", "Source format for every input (dbml, dctx, drawdb, graphql, json, yaml, gorm, bun, drizzle, prisma, typeorm, sqlite)")
+	batchCmd.Flags().StringSliceVar(&batchInputs, "input", nil, "Input file path or glob pattern (repeatable, comma-separated)")
+	batchCmd.Flags().StringVar(&batchTargetType, "to", "", "Target format")
+	batchCmd.Flags().StringVar(&batchTargetDir, "to-dir", "", "Output directory; one output per input is written here")
+	batchCmd.Flags().StringVar(&batchPackageName, "package", "", "Package name (for code generation formats like gorm/bun)")
+	batchCmd.Flags().StringVar(&batchSchemaFilter, "schema", "", "Filter to a specific schema by name")
+	batchCmd.Flags().BoolVar(&batchFlattenSchema, "flatten-schema", false, "Flatten schema.table names to schema_table")
+	batchCmd.Flags().StringVar(&batchNullableTypes, "types", "", "Nullable type package for code-gen writers (bun/gorm)")
+	batchCmd.Flags().StringVar(&batchNullableArrays, "array-nullable", "", "Nullable array representation for the Bun writer")
+	batchCmd.Flags().BoolVar(&batchContinueOnError, "continue-on-error", false, "Prepend \\set ON_ERROR_STOP off to generated SQL (pgsql output only)")
+	batchCmd.Flags().BoolVar(&batchKeepGoing, "keep-going", false, "Process remaining inputs after a failure; still exits non-zero")
+	batchCmd.Flags().BoolVar(&batchDryRun, "dry-run", false, "Read and validate every input and print the plan without writing any output")
+
+	for _, f := range []string{"from", "input", "to", "to-dir"} {
+		if err := batchCmd.MarkFlagRequired(f); err != nil {
+			fmt.Fprintf(os.Stderr, "Error marking %s flag as required: %v\n", f, err)
+		}
+	}
+}
+
+// batchDirTargets are writers that emit a directory of files rather than one file.
+var batchDirTargets = map[string]bool{"gorm": true, "bun": true, "drizzle": true}
+
+// batchExtensions maps single-file target formats to their output extension.
+var batchExtensions = map[string]string{
+	"dbml": ".dbml", "dctx": ".dctx", "drawdb": ".ddb", "json": ".json",
+	"yaml": ".yaml", "yml": ".yaml", "pgsql": ".sql", "postgres": ".sql",
+	"postgresql": ".sql", "sql": ".sql", "mssql": ".sql", "sqlserver": ".sql",
+	"mssql2016": ".sql", "mssql2017": ".sql", "mssql2019": ".sql", "mssql2022": ".sql",
+	"sqlite": ".sql", "sqlite3": ".sql", "prisma": ".prisma", "typeorm": ".ts",
+	"graphql": ".graphql", "gql": ".graphql",
+}
+
+// expandBatchInputs resolves paths and glob patterns into a sorted,
+// de-duplicated file list. A pattern that matches nothing is an error.
+func expandBatchInputs(patterns []string) ([]string, error) {
+	seen := map[string]bool{}
+	var files []string
+	for _, p := range patterns {
+		p = strings.TrimSpace(p)
+		if p == "" {
+			continue
+		}
+		matches, err := filepath.Glob(p)
+		if err != nil {
+			return nil, fmt.Errorf("invalid pattern %q: %w", p, err)
+		}
+		if len(matches) == 0 {
+			return nil, fmt.Errorf("no files match %q", p)
+		}
+		for _, m := range matches {
+			if !seen[m] {
+				seen[m] = true
+				files = append(files, m)
+			}
+		}
+	}
+	if len(files) == 0 {
+		return nil, fmt.Errorf("no input files given")
+	}
+	sort.Strings(files)
+	return files, nil
+}
+
+// batchOutputPaths returns the output path for each input. It errors when two
+// inputs would collide on the same output name.
+func batchOutputPaths(files []string, targetType, dir string) ([]string, error) {
+	key := strings.ToLower(targetType)
+	ext := ""
+	if !batchDirTargets[key] {
+		var ok bool
+		if ext, ok = batchExtensions[key]; !ok {
+			return nil, fmt.Errorf("unsupported target format: %s", targetType)
+		}
+	}
+	outs := make([]string, len(files))
+	owner := map[string]string{}
+	for i, f := range files {
+		stem := strings.TrimSuffix(filepath.Base(f), filepath.Ext(f))
+		out := filepath.Join(dir, stem+ext)
+		if prev, dup := owner[out]; dup {
+			return nil, fmt.Errorf("inputs %s and %s would both write %s", prev, f, out)
+		}
+		owner[out] = f
+		outs[i] = out
+	}
+	return outs, nil
+}
+
+func runBatch(cmd *cobra.Command, args []string) error {
+	files, err := expandBatchInputs(batchInputs)
+	if err != nil {
+		return err
+	}
+	outs, err := batchOutputPaths(files, batchTargetType, batchTargetDir)
+	if err != nil {
+		return err
+	}
+
+	fmt.Fprintf(os.Stderr, "\n=== RelSpec Batch Converter ===\n")
+	fmt.Fprintf(os.Stderr, "Started at: %s\n", getCurrentTimestamp())
+	fmt.Fprintf(os.Stderr, "Inputs: %d file(s), %s -> %s\n\n", len(files), batchSourceType, batchTargetType)
+
+	out := outWriter(cmd)
+	if batchDryRun {
+		fmt.Fprintf(out, "RelSpec batch plan (dry run - nothing written):\n")
+	}
+
+	var failed []string
+	for i, f := range files {
+		fmt.Fprintf(os.Stderr, "[%d/%d] %s -> %s\n", i+1, len(files), f, outs[i])
+		if err := processBatchItem(cmd, f, outs[i]); err != nil {
+			fmt.Fprintf(os.Stderr, "      ✗ %v\n", err)
+			failed = append(failed, fmt.Sprintf("%s: %v", f, err))
+			if !batchKeepGoing {
+				break
+			}
+			continue
+		}
+		fmt.Fprintf(os.Stderr, "      ✓ done\n")
+	}
+
+	fmt.Fprintf(os.Stderr, "\n=== Batch Complete: %d ok, %d failed ===\n", len(files)-len(failed), len(failed))
+	if len(failed) > 0 {
+		return fmt.Errorf("batch finished with %d failure(s):\n  %s", len(failed), strings.Join(failed, "\n  "))
+	}
+	return nil
+}
+
+func processBatchItem(cmd *cobra.Command, in, outPath string) error {
+	db, err := readDatabaseForConvert(batchSourceType, in, "")
+	if err != nil {
+		return fmt.Errorf("failed to read source: %w", err)
+	}
+	finalizeCommentedRefs(db, stderrWarn)
+
+	if batchDryRun {
+		if err := validateWriteTarget(db, batchTargetType, batchPackageName, batchSchemaFilter, ""); err != nil {
+			return fmt.Errorf("dry run validation failed: %w", err)
+		}
+		w := outWriter(cmd)
+		fmt.Fprintf(w, "  %s -> %s (database '%s')\n", in, outPath, db.Name)
+		printDryRunPlan(w, db)
+		return nil
+	}
+
+	if err := os.MkdirAll(batchTargetDir, 0o755); err != nil {
+		return fmt.Errorf("failed to create output directory: %w", err)
+	}
+	if err := writeDatabase(db, batchTargetType, outPath, batchPackageName, batchSchemaFilter, batchFlattenSchema, batchNullableTypes, batchNullableArrays, batchContinueOnError, ""); err != nil {
+		return fmt.Errorf("failed to write target: %w", err)
+	}
+	return nil
+}
@@ -0,0 +1,143 @@
+package main
+
+import (
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+)
+
+func saveBatchState(t *testing.T) {
+	t.Helper()
+	a, b, c, d, e, f, g := batchSourceType, batchInputs, batchTargetType, batchTargetDir, batchPackageName, batchKeepGoing, batchDryRun
+	t.Cleanup(func() {
+		batchSourceType, batchInputs, batchTargetType, batchTargetDir, batchPackageName, batchKeepGoing, batchDryRun = a, b, c, d, e, f, g
+	})
+}
+
+func TestRunBatch_ConvertsEachInput(t *testing.T) {
+	saveBatchState(t)
+	dir := t.TempDir()
+	writeTestJSON(t, filepath.Join(dir, "a.json"), []string{"users"})
+	writeTestJSON(t, filepath.Join(dir, "b.json"), []string{"posts"})
+	outDir := filepath.Join(dir, "out")
+
+	batchSourceType, batchTargetType, batchTargetDir = "json", "yaml", outDir
+	batchPackageName, batchKeepGoing, batchDryRun = "", false, false
+	batchInputs = []string{filepath.Join(dir, "*.json")}
+
+	cmd, _ := newDryRunCmd()
+	if err := runBatch(cmd, nil); err != nil {
+		t.Fatalf("batch: %v", err)
+	}
+	for _, name := range []string{"a.yaml", "b.yaml"} {
+		if _, err := os.Stat(filepath.Join(outDir, name)); err != nil {
+			t.Errorf("expected %s: %v", name, err)
+		}
+	}
+}
+
+func TestRunBatch_DryRunWritesNothing(t *testing.T) {
+	saveBatchState(t)
+	dir := t.TempDir()
+	writeTestJSON(t, filepath.Join(dir, "a.json"), []string{"users"})
+	outDir := filepath.Join(dir, "out")
+
+	batchSourceType, batchTargetType, batchTargetDir = "json", "yaml", outDir
+	batchPackageName, batchKeepGoing, batchDryRun = "", false, true
+	batchInputs = []string{filepath.Join(dir, "a.json")}
+
+	cmd, buf := newDryRunCmd()
+	if err := runBatch(cmd, nil); err != nil {
+		t.Fatalf("dry run: %v", err)
+	}
+	if _, err := os.Stat(outDir); !os.IsNotExist(err) {
+		t.Fatal("dry run must not create the output directory")
+	}
+	if !strings.Contains(buf.String(), "users") || !strings.Contains(buf.String(), "a.yaml") {
+		t.Errorf("plan incomplete:\n%s", buf.String())
+	}
+}
+
+func TestRunBatch_FailureHandling(t *testing.T) {
+	saveBatchState(t)
+	dir := t.TempDir()
+	writeTestJSON(t, filepath.Join(dir, "a.json"), []string{"users"})
+	if err := os.WriteFile(filepath.Join(dir, "b.json"), []byte("{not json"), 0o644); err != nil {
+		t.Fatal(err)
+	}
+	writeTestJSON(t, filepath.Join(dir, "c.json"), []string{"posts"})
+	outDir := filepath.Join(dir, "out")
+
+	batchSourceType, batchTargetType, batchTargetDir = "json", "yaml", outDir
+	batchPackageName, batchDryRun = "", false
+	batchInputs = []string{filepath.Join(dir, "*.json")}
+	cmd, _ := newDryRunCmd()
+
+	// Default: stop at first failure.
+	batchKeepGoing = false
+	err := runBatch(cmd, nil)
+	if err == nil || !strings.Contains(err.Error(), "b.json") {
+		t.Fatalf("expected failure naming b.json, got %v", err)
+	}
+	if _, statErr := os.Stat(filepath.Join(outDir, "c.yaml")); !os.IsNotExist(statErr) {
+		t.Error("c.json should not be processed without --keep-going")
+	}
+
+	// --keep-going: remaining inputs are processed, exit still fails.
+	batchKeepGoing = true
+	if err := runBatch(cmd, nil); err == nil {
+		t.Fatal("expected non-zero result with --keep-going")
+	}
+	if _, statErr := os.Stat(filepath.Join(outDir, "c.yaml")); statErr != nil {
+		t.Errorf("c.yaml should be written with --keep-going: %v", statErr)
+	}
+}
+
+func TestExpandBatchInputs(t *testing.T) {
+	dir := t.TempDir()
+	for _, n := range []string{"b.json", "a.json"} {
+		if err := os.WriteFile(filepath.Join(dir, n), []byte("{}"), 0o644); err != nil {
+			t.Fatal(err)
+		}
+	}
+	got, err := expandBatchInputs([]string{filepath.Join(dir, "*.json"), filepath.Join(dir, "a.json")})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(got) != 2 || filepath.Base(got[0]) != "a.json" || filepath.Base(got[1]) != "b.json" {
+		t.Errorf("want sorted deduped [a b], got %v", got)
+	}
+	if _, err := expandBatchInputs([]string{filepath.Join(dir, "*.nope")}); err == nil {
+		t.Error("unmatched pattern should error")
+	}
+}
+
+func TestBatchOutputPaths(t *testing.T) {
+	tests := []struct {
+		name    string
+		files   []string
+		target  string
+		want    []string
+		wantErr string
+	}{
+		{"file target", []string{"x/a.dbml"}, "json", []string{"out/a.json"}, ""},
+		{"dir target", []string{"x/a.json"}, "gorm", []string{"out/a"}, ""},
+		{"collision", []string{"x/a.json", "y/a.json"}, "yaml", nil, "both write"},
+		{"unsupported", []string{"a.json"}, "nope", nil, "unsupported target"},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got, err := batchOutputPaths(tt.files, tt.target, "out")
+			if tt.wantErr != "" {
+				if err == nil || !strings.Contains(err.Error(), tt.wantErr) {
+					t.Fatalf("want error %q, got %v", tt.wantErr, err)
+				}
+				return
+			}
+			if err != nil || len(got) != len(tt.want) || got[0] != filepath.FromSlash(tt.want[0]) {
+				t.Fatalf("got %v, %v; want %v", got, err, tt.want)
+			}
+		})
+	}
+}
@@ -40,6 +40,7 @@ JSON, YAML, SQL, etc.).`,

 func init() {
 	rootCmd.AddCommand(convertCmd)
+	rootCmd.AddCommand(batchCmd)
 	rootCmd.AddCommand(diffCmd)
 	rootCmd.AddCommand(inspectCmd)
 	rootCmd.AddCommand(scriptsCmd)