From d961536186b9069f52d6f8d9513f492d0bcd2864 Mon Sep 17 00:00:00 2001 From: SG Command Date: Sat, 3 Oct 2026 11:28:57 +0200 Subject: [PATCH] feat(cli): add batch command to convert many inputs in one run Co-Authored-By: Claude Sonnet 5.5 --- README.md | 11 ++ cmd/relspec/batch.go | 214 ++++++++++++++++++++++++++++++++++++++ cmd/relspec/batch_test.go | 143 +++++++++++++++++++++++++ cmd/relspec/root.go | 1 + 4 files changed, 369 insertions(+) create mode 100644 cmd/relspec/batch.go create mode 100644 cmd/relspec/batch_test.go diff --git a/README.md b/README.md index 6d43f99..a18672f 100644 --- a/README.md +++ b/README.md @@ -44,6 +44,17 @@ relspec convert --from pgsql --from-conn "postgres://..." --to sqlite --to-path relspec convert --from json --from-list "a.json,b.json" --to yaml --to-path merged.yaml ``` +### `batch` — Convert many inputs in one run + +Converts each input independently (one output per input, unlike `--from-list` +which merges). `--input` takes paths or globs; outputs go to `--to-dir`. +Use `--keep-going` to continue past failures (exit is still non-zero) and +`--dry-run` to validate without writing. For named workflows see `relspec job run`. + +```bash +relspec batch --from dbml --input "schemas/*.dbml" --to json --to-dir out/ +``` + PostgreSQL connections opened by relspec set `application_name` by default to `relspecgo/` (with component suffixes internally, e.g. readers/writers). If you need a custom value, provide `application_name` explicitly in the connection diff --git a/cmd/relspec/batch.go b/cmd/relspec/batch.go new file mode 100644 index 0000000..925405e --- /dev/null +++ b/cmd/relspec/batch.go @@ -0,0 +1,214 @@ +package main + +import ( + "fmt" + "os" + "path/filepath" + "sort" + "strings" + + "github.com/spf13/cobra" +) + +var ( + batchSourceType string + batchInputs []string + batchTargetType string + batchTargetDir string + batchPackageName string + batchSchemaFilter string + batchFlattenSchema bool + batchNullableTypes string + batchNullableArrays string + batchContinueOnError bool + batchKeepGoing bool + batchDryRun bool +) + +var batchCmd = &cobra.Command{ + Use: "batch", + Short: "Convert many input files to a target format in one run", + Long: `Convert each input file independently to the target format. + +Unlike 'convert --from-list', which merges all inputs into one output, batch +mode writes one output per input into --to-dir. The output is named after the +input file (without its extension). Directory-style targets (gorm, bun, +drizzle) get a sub-directory per input. + +Inputs are given with --input, which accepts file paths and glob patterns and +may be repeated or comma-separated. Inputs are processed in sorted order and +duplicates are removed. The command exits non-zero if any input fails. + +For named, multi-step workflows use 'relspec job run' instead. + +Examples: + # Convert every DBML file in a directory to JSON + relspec batch --from dbml --input "schemas/*.dbml" --to json --to-dir out/ + + # Convert specific files to GORM models, one package directory per input + relspec batch --from json --input a.json,b.json \ + --to gorm --to-dir models/ --package models + + # Validate everything first, writing nothing + relspec batch --from yaml --input "specs/*.yaml" --to pgsql --to-dir sql/ --dry-run + + # Report all failures instead of stopping at the first + relspec batch --from json --input "*.json" --to yaml --to-dir out/ --keep-going`, + RunE: runBatch, +} + +func init() { + batchCmd.Flags().StringVar(&batchSourceType, "from", "", "Source format for every input (dbml, dctx, drawdb, graphql, json, yaml, gorm, bun, drizzle, prisma, typeorm, sqlite)") + batchCmd.Flags().StringSliceVar(&batchInputs, "input", nil, "Input file path or glob pattern (repeatable, comma-separated)") + batchCmd.Flags().StringVar(&batchTargetType, "to", "", "Target format") + batchCmd.Flags().StringVar(&batchTargetDir, "to-dir", "", "Output directory; one output per input is written here") + batchCmd.Flags().StringVar(&batchPackageName, "package", "", "Package name (for code generation formats like gorm/bun)") + batchCmd.Flags().StringVar(&batchSchemaFilter, "schema", "", "Filter to a specific schema by name") + batchCmd.Flags().BoolVar(&batchFlattenSchema, "flatten-schema", false, "Flatten schema.table names to schema_table") + batchCmd.Flags().StringVar(&batchNullableTypes, "types", "", "Nullable type package for code-gen writers (bun/gorm)") + batchCmd.Flags().StringVar(&batchNullableArrays, "array-nullable", "", "Nullable array representation for the Bun writer") + batchCmd.Flags().BoolVar(&batchContinueOnError, "continue-on-error", false, "Prepend \\set ON_ERROR_STOP off to generated SQL (pgsql output only)") + batchCmd.Flags().BoolVar(&batchKeepGoing, "keep-going", false, "Process remaining inputs after a failure; still exits non-zero") + batchCmd.Flags().BoolVar(&batchDryRun, "dry-run", false, "Read and validate every input and print the plan without writing any output") + + for _, f := range []string{"from", "input", "to", "to-dir"} { + if err := batchCmd.MarkFlagRequired(f); err != nil { + fmt.Fprintf(os.Stderr, "Error marking %s flag as required: %v\n", f, err) + } + } +} + +// batchDirTargets are writers that emit a directory of files rather than one file. +var batchDirTargets = map[string]bool{"gorm": true, "bun": true, "drizzle": true} + +// batchExtensions maps single-file target formats to their output extension. +var batchExtensions = map[string]string{ + "dbml": ".dbml", "dctx": ".dctx", "drawdb": ".ddb", "json": ".json", + "yaml": ".yaml", "yml": ".yaml", "pgsql": ".sql", "postgres": ".sql", + "postgresql": ".sql", "sql": ".sql", "mssql": ".sql", "sqlserver": ".sql", + "mssql2016": ".sql", "mssql2017": ".sql", "mssql2019": ".sql", "mssql2022": ".sql", + "sqlite": ".sql", "sqlite3": ".sql", "prisma": ".prisma", "typeorm": ".ts", + "graphql": ".graphql", "gql": ".graphql", +} + +// expandBatchInputs resolves paths and glob patterns into a sorted, +// de-duplicated file list. A pattern that matches nothing is an error. +func expandBatchInputs(patterns []string) ([]string, error) { + seen := map[string]bool{} + var files []string + for _, p := range patterns { + p = strings.TrimSpace(p) + if p == "" { + continue + } + matches, err := filepath.Glob(p) + if err != nil { + return nil, fmt.Errorf("invalid pattern %q: %w", p, err) + } + if len(matches) == 0 { + return nil, fmt.Errorf("no files match %q", p) + } + for _, m := range matches { + if !seen[m] { + seen[m] = true + files = append(files, m) + } + } + } + if len(files) == 0 { + return nil, fmt.Errorf("no input files given") + } + sort.Strings(files) + return files, nil +} + +// batchOutputPaths returns the output path for each input. It errors when two +// inputs would collide on the same output name. +func batchOutputPaths(files []string, targetType, dir string) ([]string, error) { + key := strings.ToLower(targetType) + ext := "" + if !batchDirTargets[key] { + var ok bool + if ext, ok = batchExtensions[key]; !ok { + return nil, fmt.Errorf("unsupported target format: %s", targetType) + } + } + outs := make([]string, len(files)) + owner := map[string]string{} + for i, f := range files { + stem := strings.TrimSuffix(filepath.Base(f), filepath.Ext(f)) + out := filepath.Join(dir, stem+ext) + if prev, dup := owner[out]; dup { + return nil, fmt.Errorf("inputs %s and %s would both write %s", prev, f, out) + } + owner[out] = f + outs[i] = out + } + return outs, nil +} + +func runBatch(cmd *cobra.Command, args []string) error { + files, err := expandBatchInputs(batchInputs) + if err != nil { + return err + } + outs, err := batchOutputPaths(files, batchTargetType, batchTargetDir) + if err != nil { + return err + } + + fmt.Fprintf(os.Stderr, "\n=== RelSpec Batch Converter ===\n") + fmt.Fprintf(os.Stderr, "Started at: %s\n", getCurrentTimestamp()) + fmt.Fprintf(os.Stderr, "Inputs: %d file(s), %s -> %s\n\n", len(files), batchSourceType, batchTargetType) + + out := outWriter(cmd) + if batchDryRun { + fmt.Fprintf(out, "RelSpec batch plan (dry run - nothing written):\n") + } + + var failed []string + for i, f := range files { + fmt.Fprintf(os.Stderr, "[%d/%d] %s -> %s\n", i+1, len(files), f, outs[i]) + if err := processBatchItem(cmd, f, outs[i]); err != nil { + fmt.Fprintf(os.Stderr, " ✗ %v\n", err) + failed = append(failed, fmt.Sprintf("%s: %v", f, err)) + if !batchKeepGoing { + break + } + continue + } + fmt.Fprintf(os.Stderr, " ✓ done\n") + } + + fmt.Fprintf(os.Stderr, "\n=== Batch Complete: %d ok, %d failed ===\n", len(files)-len(failed), len(failed)) + if len(failed) > 0 { + return fmt.Errorf("batch finished with %d failure(s):\n %s", len(failed), strings.Join(failed, "\n ")) + } + return nil +} + +func processBatchItem(cmd *cobra.Command, in, outPath string) error { + db, err := readDatabaseForConvert(batchSourceType, in, "") + if err != nil { + return fmt.Errorf("failed to read source: %w", err) + } + finalizeCommentedRefs(db, stderrWarn) + + if batchDryRun { + if err := validateWriteTarget(db, batchTargetType, batchPackageName, batchSchemaFilter, ""); err != nil { + return fmt.Errorf("dry run validation failed: %w", err) + } + w := outWriter(cmd) + fmt.Fprintf(w, " %s -> %s (database '%s')\n", in, outPath, db.Name) + printDryRunPlan(w, db) + return nil + } + + if err := os.MkdirAll(batchTargetDir, 0o755); err != nil { + return fmt.Errorf("failed to create output directory: %w", err) + } + if err := writeDatabase(db, batchTargetType, outPath, batchPackageName, batchSchemaFilter, batchFlattenSchema, batchNullableTypes, batchNullableArrays, batchContinueOnError, ""); err != nil { + return fmt.Errorf("failed to write target: %w", err) + } + return nil +} diff --git a/cmd/relspec/batch_test.go b/cmd/relspec/batch_test.go new file mode 100644 index 0000000..5f1adcf --- /dev/null +++ b/cmd/relspec/batch_test.go @@ -0,0 +1,143 @@ +package main + +import ( + "os" + "path/filepath" + "strings" + "testing" +) + +func saveBatchState(t *testing.T) { + t.Helper() + a, b, c, d, e, f, g := batchSourceType, batchInputs, batchTargetType, batchTargetDir, batchPackageName, batchKeepGoing, batchDryRun + t.Cleanup(func() { + batchSourceType, batchInputs, batchTargetType, batchTargetDir, batchPackageName, batchKeepGoing, batchDryRun = a, b, c, d, e, f, g + }) +} + +func TestRunBatch_ConvertsEachInput(t *testing.T) { + saveBatchState(t) + dir := t.TempDir() + writeTestJSON(t, filepath.Join(dir, "a.json"), []string{"users"}) + writeTestJSON(t, filepath.Join(dir, "b.json"), []string{"posts"}) + outDir := filepath.Join(dir, "out") + + batchSourceType, batchTargetType, batchTargetDir = "json", "yaml", outDir + batchPackageName, batchKeepGoing, batchDryRun = "", false, false + batchInputs = []string{filepath.Join(dir, "*.json")} + + cmd, _ := newDryRunCmd() + if err := runBatch(cmd, nil); err != nil { + t.Fatalf("batch: %v", err) + } + for _, name := range []string{"a.yaml", "b.yaml"} { + if _, err := os.Stat(filepath.Join(outDir, name)); err != nil { + t.Errorf("expected %s: %v", name, err) + } + } +} + +func TestRunBatch_DryRunWritesNothing(t *testing.T) { + saveBatchState(t) + dir := t.TempDir() + writeTestJSON(t, filepath.Join(dir, "a.json"), []string{"users"}) + outDir := filepath.Join(dir, "out") + + batchSourceType, batchTargetType, batchTargetDir = "json", "yaml", outDir + batchPackageName, batchKeepGoing, batchDryRun = "", false, true + batchInputs = []string{filepath.Join(dir, "a.json")} + + cmd, buf := newDryRunCmd() + if err := runBatch(cmd, nil); err != nil { + t.Fatalf("dry run: %v", err) + } + if _, err := os.Stat(outDir); !os.IsNotExist(err) { + t.Fatal("dry run must not create the output directory") + } + if !strings.Contains(buf.String(), "users") || !strings.Contains(buf.String(), "a.yaml") { + t.Errorf("plan incomplete:\n%s", buf.String()) + } +} + +func TestRunBatch_FailureHandling(t *testing.T) { + saveBatchState(t) + dir := t.TempDir() + writeTestJSON(t, filepath.Join(dir, "a.json"), []string{"users"}) + if err := os.WriteFile(filepath.Join(dir, "b.json"), []byte("{not json"), 0o644); err != nil { + t.Fatal(err) + } + writeTestJSON(t, filepath.Join(dir, "c.json"), []string{"posts"}) + outDir := filepath.Join(dir, "out") + + batchSourceType, batchTargetType, batchTargetDir = "json", "yaml", outDir + batchPackageName, batchDryRun = "", false + batchInputs = []string{filepath.Join(dir, "*.json")} + cmd, _ := newDryRunCmd() + + // Default: stop at first failure. + batchKeepGoing = false + err := runBatch(cmd, nil) + if err == nil || !strings.Contains(err.Error(), "b.json") { + t.Fatalf("expected failure naming b.json, got %v", err) + } + if _, statErr := os.Stat(filepath.Join(outDir, "c.yaml")); !os.IsNotExist(statErr) { + t.Error("c.json should not be processed without --keep-going") + } + + // --keep-going: remaining inputs are processed, exit still fails. + batchKeepGoing = true + if err := runBatch(cmd, nil); err == nil { + t.Fatal("expected non-zero result with --keep-going") + } + if _, statErr := os.Stat(filepath.Join(outDir, "c.yaml")); statErr != nil { + t.Errorf("c.yaml should be written with --keep-going: %v", statErr) + } +} + +func TestExpandBatchInputs(t *testing.T) { + dir := t.TempDir() + for _, n := range []string{"b.json", "a.json"} { + if err := os.WriteFile(filepath.Join(dir, n), []byte("{}"), 0o644); err != nil { + t.Fatal(err) + } + } + got, err := expandBatchInputs([]string{filepath.Join(dir, "*.json"), filepath.Join(dir, "a.json")}) + if err != nil { + t.Fatal(err) + } + if len(got) != 2 || filepath.Base(got[0]) != "a.json" || filepath.Base(got[1]) != "b.json" { + t.Errorf("want sorted deduped [a b], got %v", got) + } + if _, err := expandBatchInputs([]string{filepath.Join(dir, "*.nope")}); err == nil { + t.Error("unmatched pattern should error") + } +} + +func TestBatchOutputPaths(t *testing.T) { + tests := []struct { + name string + files []string + target string + want []string + wantErr string + }{ + {"file target", []string{"x/a.dbml"}, "json", []string{"out/a.json"}, ""}, + {"dir target", []string{"x/a.json"}, "gorm", []string{"out/a"}, ""}, + {"collision", []string{"x/a.json", "y/a.json"}, "yaml", nil, "both write"}, + {"unsupported", []string{"a.json"}, "nope", nil, "unsupported target"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := batchOutputPaths(tt.files, tt.target, "out") + if tt.wantErr != "" { + if err == nil || !strings.Contains(err.Error(), tt.wantErr) { + t.Fatalf("want error %q, got %v", tt.wantErr, err) + } + return + } + if err != nil || len(got) != len(tt.want) || got[0] != filepath.FromSlash(tt.want[0]) { + t.Fatalf("got %v, %v; want %v", got, err, tt.want) + } + }) + } +} diff --git a/cmd/relspec/root.go b/cmd/relspec/root.go index 13de477..41c68d4 100644 --- a/cmd/relspec/root.go +++ b/cmd/relspec/root.go @@ -32,6 +32,7 @@ JSON, YAML, SQL, etc.).`, func init() { rootCmd.AddCommand(convertCmd) + rootCmd.AddCommand(batchCmd) rootCmd.AddCommand(diffCmd) rootCmd.AddCommand(inspectCmd) rootCmd.AddCommand(scriptsCmd)