From 2b6bb7f9483ccede5cbcf5408c0213ed59ad28b9 Mon Sep 17 00:00:00 2001 From: Hein Date: Mon, 24 Aug 2026 11:52:24 +0200 Subject: [PATCH] fix(sqlite): emit inline foreign keys, bare-name default schema, direct exec SQLite can't ALTER TABLE ADD CONSTRAINT, so foreign keys are now written as inline FOREIGN KEY clauses in CREATE TABLE instead of commented-out ALTER statements. The default schema (public/main) now produces bare table names instead of a "public_" prefix; other schemas are still prefixed to avoid collisions. Also adds direct-to-file execution: the sqlite writer can now apply generated DDL straight to a .db file via Metadata["connection_string"], wired into `relspec merge --output-conn`. --- cmd/relspec/merge.go | 12 +- pkg/writers/sqlite/README.md | 59 ++++--- pkg/writers/sqlite/templates.go | 62 +++++-- .../sqlite/templates/create_foreign_key.tmpl | 6 - .../sqlite/templates/create_table.tmpl | 3 + pkg/writers/sqlite/writer.go | 166 ++++++++++++------ pkg/writers/sqlite/writer_test.go | 15 +- 7 files changed, 217 insertions(+), 106 deletions(-) delete mode 100644 pkg/writers/sqlite/templates/create_foreign_key.tmpl diff --git a/cmd/relspec/merge.go b/cmd/relspec/merge.go index 70aa66f..203e3c4 100644 --- a/cmd/relspec/merge.go +++ b/cmd/relspec/merge.go @@ -117,7 +117,7 @@ func init() { // Output flags mergeCmd.Flags().StringVar(&mergeOutputType, "output", "", "Output format (required): dbml, dctx, drawdb, graphql, json, yaml, gorm, bun, drizzle, prisma, typeorm, pgsql") mergeCmd.Flags().StringVar(&mergeOutputPath, "output-path", "", "Output file path (required for file-based formats)") - mergeCmd.Flags().StringVar(&mergeOutputConn, "output-conn", "", "Output connection string (for pgsql)") + mergeCmd.Flags().StringVar(&mergeOutputConn, "output-conn", "", "Output connection string (for pgsql) or database file path (for sqlite, to execute DDL directly instead of writing a .sql file)") // Merge options mergeCmd.Flags().BoolVar(&mergeSkipDomains, "skip-domains", false, "Skip domains during merge") @@ -427,7 +427,15 @@ func writeDatabaseForMerge(dbType, filePath, connString string, db *models.Datab } writer = wtypeorm.NewWriter(newWriterOptions(filePath, "", flattenSchema, "", "", false)) case "sqlite", "sqlite3": - writer = wsqlite.NewWriter(newWriterOptions(filePath, "", flattenSchema, "", "", false)) + writerOpts := newWriterOptions(filePath, "", flattenSchema, "", "", false) + if connString != "" { + // Execute DDL directly against the SQLite database file instead + // of writing a .sql script. + writerOpts.Metadata = map[string]interface{}{ + "connection_string": connString, + } + } + writer = wsqlite.NewWriter(writerOpts) case "pgsql": writerOpts := newWriterOptions(filePath, "", flattenSchema, "", "", false) if connString != "" { diff --git a/pkg/writers/sqlite/README.md b/pkg/writers/sqlite/README.md index 2edbd31..8bf6fb5 100644 --- a/pkg/writers/sqlite/README.md +++ b/pkg/writers/sqlite/README.md @@ -4,13 +4,14 @@ SQLite DDL (Data Definition Language) writer for RelSpec. Converts database sche ## Features -- **Automatic Schema Flattening** - SQLite doesn't support PostgreSQL-style schemas, so table names are automatically flattened (e.g., `public.users` → `public_users`) +- **Schema Flattening** - SQLite doesn't support PostgreSQL-style schemas. Non-default schema names are flattened into table name prefixes (e.g., `auth.sessions` → `auth_sessions`); the default schema (`public`/`main`) is left as bare table names (e.g., `public.users` → `users`) - **Type Mapping** - Converts PostgreSQL data types to SQLite type affinities (TEXT, INTEGER, REAL, NUMERIC, BLOB) - **Auto-Increment Detection** - Automatically converts SERIAL types and auto-increment columns to `INTEGER PRIMARY KEY AUTOINCREMENT` - **Function Translation** - Converts PostgreSQL functions to SQLite equivalents (e.g., `now()` → `CURRENT_TIMESTAMP`) - **Boolean Handling** - Maps boolean values to INTEGER (true=1, false=0) -- **Constraint Generation** - Creates indexes, unique constraints, and documents foreign keys +- **Constraint Generation** - Creates indexes, unique constraints, and inline `FOREIGN KEY` clauses in `CREATE TABLE` - **Identifier Quoting** - Properly quotes identifiers using double quotes +- **Direct Execution** - Can execute the generated DDL directly against a `.db` file instead of writing a `.sql` script (see below) ## Usage @@ -30,15 +31,26 @@ relspec convert --from dbml --from-path schema.dbml \ ### Multi-Schema Databases -SQLite doesn't support schemas, so multi-schema databases are automatically flattened: +SQLite doesn't support schemas, so multi-schema databases are automatically flattened. The default schema (`public`/`main`) keeps bare table names; other schemas are prefixed to avoid collisions: ```bash # Input has auth.users and public.posts -# Output will have auth_users and public_posts +# Output will have auth_users and posts relspec convert --from json --from-path multi_schema.json \ --to sqlite --to-path flattened.sql ``` +### Direct Execution Against a Database File + +`relspec merge` can execute the generated DDL directly against a SQLite file instead of writing a `.sql` script, by passing the file path as `--output-conn`: + +```bash +relspec merge --source dbml --source-path schema.dbml \ + --output sqlite --output-conn ./app.db +``` + +Passing `--output-conn` opens `./app.db` and applies the schema directly; passing `--output-path` instead (or omitting `--output-conn`) writes a `.sql` script as before. + ## Type Mapping | PostgreSQL Type | SQLite Affinity | Examples | @@ -87,17 +99,17 @@ CREATE TABLE "users" ( ## Foreign Keys -Foreign keys are generated as commented-out ALTER TABLE statements for reference: +SQLite has no `ALTER TABLE ADD CONSTRAINT`, so foreign keys are generated as inline `FOREIGN KEY` clauses inside `CREATE TABLE`, exactly as SQLite requires: ```sql --- Foreign key: fk_posts_user_id --- ALTER TABLE "posts" ADD CONSTRAINT "posts_fk_posts_user_id" --- FOREIGN KEY ("user_id") --- REFERENCES "users" ("id"); --- Note: Foreign keys should be defined in CREATE TABLE for better SQLite compatibility +CREATE TABLE "posts" ( + "id" INTEGER PRIMARY KEY AUTOINCREMENT, + "user_id" INTEGER NOT NULL, + FOREIGN KEY ("user_id") REFERENCES "users" ("id") ON DELETE CASCADE +); ``` -For production use, define foreign keys directly in the CREATE TABLE statement or execute the ALTER TABLE commands after creating all tables. +`PRAGMA foreign_keys = ON;` is emitted at the top of the output (and executed first in direct-execution mode) so these constraints are actually enforced. ## Constraints @@ -112,11 +124,10 @@ Generated SQL follows this order: 1. Header comments 2. `PRAGMA foreign_keys = ON;` -3. CREATE TABLE statements (sorted by schema, then table) +3. CREATE TABLE statements (sorted by schema, then table), with primary keys and foreign keys defined inline 4. CREATE INDEX statements 5. CREATE UNIQUE INDEX statements (for unique constraints) 6. Check constraint comments -7. Foreign key comments ## Example @@ -145,7 +156,7 @@ CREATE TABLE public.posts ( -- SQLite Database Schema -- Database: mydb -- Generated by RelSpec --- Note: Schema names have been flattened (e.g., public.users -> public_users) +-- Note: SQLite has no schema concept; non-default schema names are flattened into table name prefixes (e.g., auth.sessions -> auth_sessions) -- Enable foreign key constraints PRAGMA foreign_keys = ON; @@ -160,22 +171,17 @@ CREATE TABLE "auth_users" ( CREATE UNIQUE INDEX "auth_users_users_username_key" ON "auth_users" ("username"); --- Schema: public (flattened into table names) - -CREATE TABLE "public_posts" ( +CREATE TABLE "posts" ( "id" INTEGER PRIMARY KEY AUTOINCREMENT, "user_id" INTEGER NOT NULL, "title" TEXT NOT NULL, - "published" INTEGER DEFAULT 0 + "published" INTEGER DEFAULT 0, + FOREIGN KEY ("user_id") REFERENCES "auth_users" ("id") ); - --- Foreign key: posts_user_id_fkey --- ALTER TABLE "public_posts" ADD CONSTRAINT "public_posts_posts_user_id_fkey" --- FOREIGN KEY ("user_id") --- REFERENCES "auth_users" ("id"); --- Note: Foreign keys should be defined in CREATE TABLE for better SQLite compatibility ``` +Note that `public.posts` becomes bare `posts` (the default schema isn't prefixed), while `auth.users` becomes `auth_users` (a non-default schema is), and the foreign key to `auth_users` is defined inline rather than as a separate statement. + ## Programmatic Usage ```go @@ -208,8 +214,9 @@ func main() { ## Notes -- Schema flattening is **always enabled** for SQLite output (cannot be disabled) +- Schema flattening is **always enabled** for SQLite output (cannot be disabled); the default schema (`public`/`main`) produces bare table names, other schemas are prefixed - Constraint and index names are prefixed with the flattened table name to avoid collisions - Generated SQL is compatible with SQLite 3.x -- Foreign key constraints require `PRAGMA foreign_keys = ON;` to be enforced +- Foreign key constraints require `PRAGMA foreign_keys = ON;` to be enforced, which is emitted (and, in direct-execution mode, run) before any `CREATE TABLE` +- Setting `Metadata["connection_string"]` to a `.db` file path (or passing `--output-conn` to `relspec merge`) executes the DDL directly against that file instead of writing a `.sql` script - For complex schemas, review and test the generated SQL before use in production diff --git a/pkg/writers/sqlite/templates.go b/pkg/writers/sqlite/templates.go index d22b2a9..49304fe 100644 --- a/pkg/writers/sqlite/templates.go +++ b/pkg/writers/sqlite/templates.go @@ -40,10 +40,22 @@ func NewTemplateExecutor(opts *writers.WriterOptions) (*TemplateExecutor, error) // TableTemplateData contains data for table template type TableTemplateData struct { - Schema string - Name string - Columns []*models.Column - PrimaryKey *models.Constraint + Schema string + Name string + Columns []*models.Column + PrimaryKey *models.Constraint + ForeignKeys []ForeignKeyTemplateData +} + +// ForeignKeyTemplateData contains data for an inline FOREIGN KEY clause +type ForeignKeyTemplateData struct { + Name string + Columns []string + ForeignSchema string + ForeignTable string + ForeignColumns []string + OnDelete string + OnUpdate string } // IndexTemplateData contains data for index template @@ -120,16 +132,6 @@ func (te *TemplateExecutor) ExecuteCreateCheckConstraint(data ConstraintTemplate return buf.String(), nil } -// ExecuteCreateForeignKey executes the create foreign key template -func (te *TemplateExecutor) ExecuteCreateForeignKey(data ConstraintTemplateData) (string, error) { - var buf bytes.Buffer - err := te.templates.ExecuteTemplate(&buf, "create_foreign_key.tmpl", data) - if err != nil { - return "", fmt.Errorf("failed to execute create_foreign_key template: %w", err) - } - return buf.String(), nil -} - // Helper functions to build template data from models // BuildTableTemplateData builds TableTemplateData from a models.Table @@ -162,11 +164,35 @@ func BuildTableTemplateData(schema string, table *models.Table) TableTemplateDat } } + // Collect foreign keys for inline FOREIGN KEY clauses + var fks []ForeignKeyTemplateData + for _, constraint := range sortConstraints(table.Constraints) { + if constraint.Type != models.ForeignKeyConstraint { + continue + } + + refSchema := tableSchemaName(constraint.ReferencedSchema) + if refSchema == "" { + refSchema = schema + } + + fks = append(fks, ForeignKeyTemplateData{ + Name: constraint.Name, + Columns: constraint.Columns, + ForeignSchema: refSchema, + ForeignTable: constraint.ReferencedTable, + ForeignColumns: constraint.ReferencedColumns, + OnDelete: constraint.OnDelete, + OnUpdate: constraint.OnUpdate, + }) + } + return TableTemplateData{ - Schema: schema, - Name: table.Name, - Columns: columns, - PrimaryKey: pk, + Schema: schema, + Name: table.Name, + Columns: columns, + PrimaryKey: pk, + ForeignKeys: fks, } } diff --git a/pkg/writers/sqlite/templates/create_foreign_key.tmpl b/pkg/writers/sqlite/templates/create_foreign_key.tmpl deleted file mode 100644 index f732322..0000000 --- a/pkg/writers/sqlite/templates/create_foreign_key.tmpl +++ /dev/null @@ -1,6 +0,0 @@ --- Foreign key: {{.Name}} --- ALTER TABLE {{quote_ident (qualified_table_name .Schema .Table)}} ADD CONSTRAINT {{quote_ident (format_constraint_name .Schema .Table .Name)}} --- FOREIGN KEY ({{range $i, $col := .Columns}}{{if $i}}, {{end}}{{quote_ident $col}}{{end}}) --- REFERENCES {{quote_ident (qualified_table_name .ForeignSchema .ForeignTable)}} ({{range $i, $col := .ForeignColumns}}{{if $i}}, {{end}}{{quote_ident $col}}{{end}}) --- {{if .OnDelete}}ON DELETE {{.OnDelete}}{{end}}{{if .OnUpdate}} ON UPDATE {{.OnUpdate}}{{end}}; --- Note: Foreign keys should be defined in CREATE TABLE for better SQLite compatibility diff --git a/pkg/writers/sqlite/templates/create_table.tmpl b/pkg/writers/sqlite/templates/create_table.tmpl index 7e1cf5f..1139d07 100644 --- a/pkg/writers/sqlite/templates/create_table.tmpl +++ b/pkg/writers/sqlite/templates/create_table.tmpl @@ -6,4 +6,7 @@ CREATE TABLE {{quote_ident (qualified_table_name .Schema .Name)}} ( {{- if and .PrimaryKey (not $hasAutoIncrement)}}{{if gt (len .Columns) 0}},{{end}} PRIMARY KEY ({{range $i, $colName := .PrimaryKey.Columns}}{{if $i}}, {{end}}{{quote_ident $colName}}{{end}}) {{- end}} +{{- range .ForeignKeys}}, + FOREIGN KEY ({{range $i, $col := .Columns}}{{if $i}}, {{end}}{{quote_ident $col}}{{end}}) REFERENCES {{quote_ident (qualified_table_name .ForeignSchema .ForeignTable)}} ({{range $i, $col := .ForeignColumns}}{{if $i}}, {{end}}{{quote_ident $col}}{{end}}){{if .OnDelete}} ON DELETE {{.OnDelete}}{{end}}{{if .OnUpdate}} ON UPDATE {{.OnUpdate}}{{end}} +{{- end}} ); diff --git a/pkg/writers/sqlite/writer.go b/pkg/writers/sqlite/writer.go index 0e96e17..7665234 100644 --- a/pkg/writers/sqlite/writer.go +++ b/pkg/writers/sqlite/writer.go @@ -1,11 +1,15 @@ package sqlite import ( + "context" + "database/sql" "fmt" "io" "os" "strings" + _ "modernc.org/sqlite" // SQLite driver + "git.warky.dev/wdevs/relspecgo/pkg/models" "git.warky.dev/wdevs/relspecgo/pkg/writers" ) @@ -30,8 +34,16 @@ func NewWriter(options *writers.WriterOptions) *Writer { } } -// WriteDatabase writes the entire database schema as SQLite SQL +// WriteDatabase writes the entire database schema as SQLite SQL. +// +// If Metadata["connection_string"] is set (a path to a SQLite database file), +// the generated DDL is executed directly against that file instead of being +// written out as a .sql script. func (w *Writer) WriteDatabase(db *models.Database) error { + if dbPath, ok := w.options.Metadata["connection_string"].(string); ok && dbPath != "" { + return w.executeDatabaseSQL(db, dbPath) + } + var writer io.Writer var file *os.File var err error @@ -52,12 +64,16 @@ func (w *Writer) WriteDatabase(db *models.Database) error { } w.writer = writer + return w.writeContent(db) +} +// writeContent writes the header, pragma, and every schema's DDL to w.writer. +func (w *Writer) writeContent(db *models.Database) error { // Write header comment fmt.Fprintf(w.writer, "-- SQLite Database Schema\n") fmt.Fprintf(w.writer, "-- Database: %s\n", db.Name) fmt.Fprintf(w.writer, "-- Generated by RelSpec\n") - fmt.Fprintf(w.writer, "-- Note: Schema names have been flattened (e.g., public.users -> public_users)\n\n") + fmt.Fprintf(w.writer, "-- Note: SQLite has no schema concept; non-default schema names are flattened into table name prefixes (e.g., auth.sessions -> auth_sessions)\n\n") // Enable foreign keys pragma, err := w.executor.ExecutePragmaForeignKeys() @@ -76,48 +92,134 @@ func (w *Writer) WriteDatabase(db *models.Database) error { return nil } +// statementCollector captures each Write call as a single SQL statement (or +// comment line), matching the writer's convention of one Fprintf per statement. +type statementCollector struct { + statements []string +} + +func (c *statementCollector) Write(p []byte) (int, error) { + if s := strings.TrimSpace(string(p)); s != "" { + c.statements = append(c.statements, s) + } + return len(p), nil +} + +// executeDatabaseSQL generates the DDL for db and executes it directly +// against the SQLite database file at dbPath. +func (w *Writer) executeDatabaseSQL(db *models.Database, dbPath string) error { + collector := &statementCollector{} + w.writer = collector + if err := w.writeContent(db); err != nil { + return fmt.Errorf("failed to generate SQL statements: %w", err) + } + + conn, err := sql.Open("sqlite", dbPath) + if err != nil { + return fmt.Errorf("failed to open sqlite database %q: %w", dbPath, err) + } + defer conn.Close() + + ctx := context.Background() + ignoreErrors := false + if val, ok := w.options.Metadata["ignore_errors"].(bool); ok { + ignoreErrors = val + } + + total, executed := 0, 0 + var execErrors []string + for _, stmt := range collector.statements { + if strings.HasPrefix(stmt, "--") { + continue + } + + total++ + if _, err := conn.ExecContext(ctx, stmt); err != nil { + execErrors = append(execErrors, fmt.Sprintf("statement %d (%s): %v", total, truncateStatement(stmt), err)) + if !ignoreErrors { + break + } + continue + } + executed++ + } + + w.options.Metadata["execution_total"] = total + w.options.Metadata["execution_success"] = executed + w.options.Metadata["execution_failed"] = len(execErrors) + + if len(execErrors) > 0 { + return fmt.Errorf("failed to execute %d/%d statement(s) against %q:\n%s", len(execErrors), total, dbPath, strings.Join(execErrors, "\n")) + } + + return nil +} + +// truncateStatement shortens a SQL statement for error messages. +func truncateStatement(stmt string) string { + const maxLen = 80 + stmt = strings.Join(strings.Fields(stmt), " ") + if len(stmt) > maxLen { + return stmt[:maxLen] + "..." + } + return stmt +} + +// defaultSchemaNames are treated as "no schema" for SQLite output: SQLite has +// no schema concept, and a lone default schema (e.g. DBML's implicit "public") +// should produce bare table names rather than a "public_" prefix. +var defaultSchemaNames = map[string]bool{ + "public": true, + "main": true, +} + +// tableSchemaName returns the schema name to use for table/constraint naming, +// collapsing default schema names to "" so they aren't prefixed onto table names. +func tableSchemaName(schema string) string { + if defaultSchemaNames[strings.ToLower(schema)] { + return "" + } + return schema +} + // WriteSchema writes a single schema as SQLite SQL func (w *Writer) WriteSchema(schema *models.Schema) error { - // SQLite doesn't have schemas, so we just write a comment - if schema.Name != "" { + tableSchema := tableSchemaName(schema.Name) + + // SQLite doesn't have schemas, so we just write a comment (skip for the + // default schema, since its tables aren't actually being prefixed) + if tableSchema != "" { fmt.Fprintf(w.writer, "-- Schema: %s (flattened into table names)\n\n", schema.Name) } // Phase 1: Create tables for _, table := range schema.Tables { - if err := w.writeTable(schema.Name, table); err != nil { + if err := w.writeTable(tableSchema, table); err != nil { return fmt.Errorf("failed to write table %s: %w", table.Name, err) } } // Phase 2: Create indexes for _, table := range schema.Tables { - if err := w.writeIndexes(schema.Name, table); err != nil { + if err := w.writeIndexes(tableSchema, table); err != nil { return fmt.Errorf("failed to write indexes for table %s: %w", table.Name, err) } } // Phase 3: Create unique constraints (as unique indexes) for _, table := range schema.Tables { - if err := w.writeUniqueConstraints(schema.Name, table); err != nil { + if err := w.writeUniqueConstraints(tableSchema, table); err != nil { return fmt.Errorf("failed to write unique constraints for table %s: %w", table.Name, err) } } // Phase 4: Check constraints (as comments, since SQLite requires them in CREATE TABLE) for _, table := range schema.Tables { - if err := w.writeCheckConstraints(schema.Name, table); err != nil { + if err := w.writeCheckConstraints(tableSchema, table); err != nil { return fmt.Errorf("failed to write check constraints for table %s: %w", table.Name, err) } } - // Phase 5: Foreign keys (as comments for compatibility) - for _, table := range schema.Tables { - if err := w.writeForeignKeys(schema.Name, table); err != nil { - return fmt.Errorf("failed to write foreign keys for table %s: %w", table.Name, err) - } - } - return nil } @@ -255,37 +357,3 @@ func (w *Writer) writeCheckConstraints(schema string, table *models.Table) error return nil } -// writeForeignKeys writes foreign keys as comments -func (w *Writer) writeForeignKeys(schema string, table *models.Table) error { - for _, constraint := range sortConstraints(table.Constraints) { - if constraint.Type != models.ForeignKeyConstraint { - continue - } - - refSchema := constraint.ReferencedSchema - if refSchema == "" { - refSchema = schema - } - - data := ConstraintTemplateData{ - Schema: schema, - Table: table.Name, - Name: constraint.Name, - Columns: constraint.Columns, - ForeignSchema: refSchema, - ForeignTable: constraint.ReferencedTable, - ForeignColumns: constraint.ReferencedColumns, - OnDelete: constraint.OnDelete, - OnUpdate: constraint.OnUpdate, - } - - sql, err := w.executor.ExecuteCreateForeignKey(data) - if err != nil { - return fmt.Errorf("failed to execute create foreign key template: %w", err) - } - - fmt.Fprintf(w.writer, "%s\n", sql) - } - - return nil -} diff --git a/pkg/writers/sqlite/writer_test.go b/pkg/writers/sqlite/writer_test.go index 640932f..aa90fba 100644 --- a/pkg/writers/sqlite/writer_test.go +++ b/pkg/writers/sqlite/writer_test.go @@ -85,8 +85,11 @@ func TestWriteDatabase(t *testing.T) { t.Error("Expected CREATE TABLE statement") } - if !strings.Contains(output, "\"public_users\"") { - t.Error("Expected flattened table name public_users") + if !strings.Contains(output, "\"users\"") { + t.Error("Expected bare table name users (default schema should not be prefixed)") + } + if strings.Contains(output, "\"public_users\"") { + t.Error("Did not expect flattened table name public_users for the default public schema") } if !strings.Contains(output, "INTEGER PRIMARY KEY AUTOINCREMENT") { @@ -322,13 +325,15 @@ func TestWriteSchema_MultiSchema(t *testing.T) { output := buf.String() - // Check for flattened table names from both schemas + // Non-default schemas are still prefixed to avoid name collisions... if !strings.Contains(output, "\"auth_sessions\"") { t.Error("Expected flattened table name auth_sessions") } - if !strings.Contains(output, "\"public_posts\"") { - t.Error("Expected flattened table name public_posts") + // ...but the default "public" schema is not, since it's typically the + // only schema and bare names read better (and match e.g. DBML output). + if !strings.Contains(output, "\"posts\"") { + t.Error("Expected bare table name posts") } }