diff --git a/README.md b/README.md index 7f03ac7..307e473 100644 --- a/README.md +++ b/README.md @@ -79,6 +79,25 @@ block contents. Raw SQL expressions remain executable input. See the [PostgreSQL writer documentation](pkg/writers/pgsql/README.md#identifier-and-value-handling) for library options and details. +#### MySQL fidelity and strict conversion + +MySQL output now preserves foreign keys and their actions, checks, indexes, +unsigned types, dimensions, generated columns, and native defaults. Known +unsupported features produce warnings. Add `--strict-conversion` to reject +lossy MySQL output before writing a file or executing DDL: + +```bash +relspec convert --from mysql --from-conn "$MYSQL_DSN" \ + --to mysql --to-path schema.sql --strict-conversion +``` + +Schema jobs can set `options.strict_conversion: true`. This check currently +covers MySQL output; it does not guarantee lossless conversion to every format. +The live suite covers MySQL 8.4 / InnoDB, including JSON/YAML intermediates. +See the [MySQL capability matrix](docs/MYSQL_FIDELITY.md) for unsupported objects, +metadata, SQL-mode requirements, and execution limits. Output creates tables in +existing databases; it is not a data copy or a rerunnable migration. + ### `batch` — Convert many inputs in one run Converts each input independently (one output per input, unlike `--from-list` @@ -202,7 +221,9 @@ references, which are expanded and safety-checked during pre-flight. See Use `options.lowercase_identifiers: true` per job, or `relspec job run build-schema --lowercase` to enable lowercasing for schema outputs -throughout the selected job plan. +throughout the selected job plan. For MySQL schema output, +`options.strict_conversion: true` or `--strict-conversion` rejects known lossy +conversions before writing. ### `scripts` — SQL scripts and embedded assets diff --git a/cmd/relspec/convert.go b/cmd/relspec/convert.go index 9853ff9..0fc0a63 100644 --- a/cmd/relspec/convert.go +++ b/cmd/relspec/convert.go @@ -181,12 +181,12 @@ Examples: } func init() { - convertCmd.Flags().StringVar(&convertSourceType, "from", "", "Source format (dbml, dctx, drawdb, graphql, json, yaml, gorm, bun, drizzle, prisma, typeorm, pgsql, sqlite)") + convertCmd.Flags().StringVar(&convertSourceType, "from", "", "Source format (dbml, dctx, drawdb, graphql, json, yaml, gorm, bun, drizzle, prisma, typeorm, pgsql, mssql, mysql, sqlite)") convertCmd.Flags().StringVar(&convertSourcePath, "from-path", "", "Source file path (for file-based formats)") - convertCmd.Flags().StringVar(&convertSourceConn, "from-conn", "", "Source connection string (for pgsql) or file path (for sqlite)") + convertCmd.Flags().StringVar(&convertSourceConn, "from-conn", "", "Source connection string (pgsql, mssql, mysql) or file path (sqlite)") convertCmd.Flags().StringSliceVar(&convertFromList, "from-list", nil, "Comma-separated list of source file paths to read and merge (mutually exclusive with --from-path)") - convertCmd.Flags().StringVar(&convertTargetType, "to", "", "Target format (dbml, dctx, drawdb, graphql, json, yaml, gorm, bun, drizzle, prisma, typeorm, pgsql)") + convertCmd.Flags().StringVar(&convertTargetType, "to", "", "Target format (dbml, dctx, drawdb, graphql, json, yaml, gorm, bun, drizzle, prisma, typeorm, pgsql, mssql, mysql, sqlite)") convertCmd.Flags().StringVar(&convertTargetPath, "to-path", "", "Target output path (file or directory)") convertCmd.Flags().StringVar(&convertPackageName, "package", "", "Package name (for code generation formats like gorm/bun)") convertCmd.Flags().StringVar(&convertSchemaFilter, "schema", "", "Filter to a specific schema by name (required for formats like dctx that only support single schemas)") @@ -449,7 +449,7 @@ func readDatabaseForConvert(dbType, filePath, connString string) (*models.Databa return db, nil } -func writeDatabase(db *models.Database, dbType, outputPath, packageName, schemaFilter string, flattenSchema bool, nullableTypes, nullableArrays string, continueOnError bool, extraFields string) error { +func writeDatabase(db *models.Database, dbType, outputPath, packageName, schemaFilter string, flattenSchema bool, nullableTypes, nullableArrays string, continueOnError bool, extraFields string, strict ...bool) error { if lowercaseIdentifiers { var err error schemaFilter = strings.ToLower(schemaFilter) @@ -462,6 +462,9 @@ func writeDatabase(db *models.Database, dbType, outputPath, packageName, schemaF var writer writers.Writer writerOpts := newWriterOptions(outputPath, packageName, flattenSchema, nullableTypes, nullableArrays, continueOnError) + if len(strict) > 0 { + writerOpts.StrictConversion = writerOpts.StrictConversion || strict[0] + } if extraFields != "" { extraFieldsJSON, err := loadExtraFields(dbType, extraFields) if err != nil { @@ -611,6 +614,19 @@ func validateWriteTarget(db *models.Database, dbType, packageName, schemaFilter, "pgsql", "postgres", "postgresql", "sql", "mssql", "sqlserver", "mssql2016", "mssql2017", "mssql2019", "mssql2022", "sqlite", "sqlite3", "prisma", "typeorm", "graphql", "gql": + case "mysql", "mariadb": + selected := db + if schemaFilter != "" { + selected = &models.Database{} + for _, s := range db.Schemas { + if s.Name == schemaFilter { + selected.Schemas = append(selected.Schemas, s) + } + } + } + if _, err := wmysql.NewWriter(newWriterOptions("", packageName, false, "", "", false)).Statements(selected); err != nil { + return err + } case "gorm", "bun": if packageName == "" { return fmt.Errorf("package name is required for %s format (use --package flag)", dbType) diff --git a/cmd/relspec/job.go b/cmd/relspec/job.go index 327aa45..19ce012 100644 --- a/cmd/relspec/job.go +++ b/cmd/relspec/job.go @@ -874,7 +874,7 @@ func writeJobOutput(rj *resolvedJob, db *models.Database, lg *jobLogger) error { lg.logf("writing output: %s (%s)", rj.outputPath, format) write := func(target string) error { - return writeDatabase(db, format, target, o.Package, o.Schema, o.FlattenSchema, o.Types, o.ArrayNullable, o.ContinueOnError, "") + return writeDatabase(db, format, target, o.Package, o.Schema, o.FlattenSchema, o.Types, o.ArrayNullable, o.ContinueOnError, "", o.StrictConversion) } // Single-file formats are written to a temp file and renamed into place so // a failure never leaves a partial or truncated output. Directory-emitting diff --git a/cmd/relspec/prisma_options.go b/cmd/relspec/prisma_options.go index 12167bf..f79b819 100644 --- a/cmd/relspec/prisma_options.go +++ b/cmd/relspec/prisma_options.go @@ -32,5 +32,6 @@ func newWriterOptions(outputPath, packageName string, flattenSchema bool, nullab Prisma7: prisma7, ContinueOnError: continueOnError, StrictDirectives: strictDirectives, + StrictConversion: strictConversion, } } diff --git a/cmd/relspec/root.go b/cmd/relspec/root.go index 9b77fc5..f9a756b 100644 --- a/cmd/relspec/root.go +++ b/cmd/relspec/root.go @@ -18,6 +18,7 @@ var ( noVersion bool silent bool strictDirectives bool + strictConversion bool lowercaseIdentifiers bool typeMapFlags []string typeMappings map[string]string @@ -62,6 +63,7 @@ func init() { rootCmd.PersistentFlags().StringArrayVar(&typeMapFlags, "type-map", nil, "Override a SQL-to-Go type mapping for bun/gorm output as sqltype=gotype (repeatable), e.g. --type-map uuid=uuid.UUID --type-map numeric=decimal.Decimal") rootCmd.PersistentFlags().BoolVar(&noVersion, "no-version", false, "Suppress the RelSpec version header") rootCmd.PersistentFlags().BoolVar(&silent, "silent", false, "Suppress progress and status messages (errors are still shown)") + rootCmd.PersistentFlags().BoolVar(&strictConversion, "strict-conversion", false, "Reject known lossy MySQL schema conversions before writing or executing output") rootCmd.PersistentFlags().BoolVar(&strictDirectives, "strict-directives", false, "Fail on unknown or untranslatable DBML dialect directives (@postgres:, @sqlite:, …)") } diff --git a/cmd/relspec/strict_conversion_test.go b/cmd/relspec/strict_conversion_test.go new file mode 100644 index 0000000..57ac502 --- /dev/null +++ b/cmd/relspec/strict_conversion_test.go @@ -0,0 +1,41 @@ +package main + +import ( + "io" + "os" + "path/filepath" + "strings" + "testing" + + "git.warky.dev/wdevs/relspecgo/pkg/jobs" + "git.warky.dev/wdevs/relspecgo/pkg/models" +) + +func TestStrictConversionOptionAndDryRun(t *testing.T) { + prior := strictConversion + strictConversion = true + t.Cleanup(func() { strictConversion = prior }) + if flag := rootCmd.PersistentFlags().Lookup("strict-conversion"); flag == nil { + t.Fatal("missing --strict-conversion") + } + db := models.InitDatabase("shop") + s := models.InitSchema("shop") + table := models.InitTable("users", "shop") + col := models.InitColumn("value", "users", "shop") + col.Type = "unknown_type" + table.Columns[col.Name] = col + s.Tables = append(s.Tables, table) + db.Schemas = append(db.Schemas, s) + if err := validateWriteTarget(db, "mysql", "", "", ""); err == nil || !strings.Contains(err.Error(), "lossy MySQL") { + t.Fatalf("dry run bypassed strict validation: %v", err) + } + strictConversion = false + out := filepath.Join(t.TempDir(), "out.sql") + rj := &resolvedJob{job: &jobs.Job{Output: &jobs.Output{Format: "mysql"}, Options: jobs.Options{StrictConversion: true}}, outputPath: out} + if err := writeJobOutput(rj, db, &jobLogger{file: io.Discard}); err == nil || !strings.Contains(err.Error(), "lossy MySQL") { + t.Fatalf("job option bypassed strict validation: %v", err) + } + if _, err := os.Stat(out); !os.IsNotExist(err) { + t.Fatalf("strict job created output: %v", err) + } +} diff --git a/docs/JOB_FILES.md b/docs/JOB_FILES.md index be4fd27..d679ed9 100644 --- a/docs/JOB_FILES.md +++ b/docs/JOB_FILES.md @@ -191,6 +191,7 @@ jobs: # (file output = full DDL); set options.full_ddl to force full DDL overwrite: false # default false options: + strict_conversion: false # reject known lossy MySQL output before writing lowercase_identifiers: false # lowercase names/references; SQL expressions and data keep their case flatten_schema: false schema: public diff --git a/docs/MYSQL_FIDELITY.md b/docs/MYSQL_FIDELITY.md new file mode 100644 index 0000000..6f2b732 --- /dev/null +++ b/docs/MYSQL_FIDELITY.md @@ -0,0 +1,131 @@ +# MySQL schema fidelity + +The MySQL reader/writer preserves the table-schema features below. It generates +DDL for **new tables in existing databases**, not migrations or data copies. +MySQL 8.4 / InnoDB is exercised by the live suite. The reader uses MySQL 8.0.23+ +metadata (including functional indexes, checks, spatial metadata, and invisible +columns); older MySQL versions and the `mariadb` alias are not certified by this +suite and may return explicit metadata-query errors. + +## Conversion diagnostics + +MySQL output warns on stderr when it encounters known unsupported features. +Add `--strict-conversion` to reject these conversions before opening an output +file or connecting to an output database: + +```bash +relspec convert --from mysql --from-conn "$MYSQL_DSN" \ + --to mysql --to-path schema.sql --strict-conversion + +relspec convert --from json --from-path schema.json \ + --to mysql --to-path schema.sql --strict-conversion --dry-run +``` + +The flag also applies to MySQL output from `batch`, `split`, and schema jobs. +Jobs can set `options.strict_conversion: true`. Library callers use +`writers.WriterOptions.StrictConversion`; `Warning func(string)` collects +non-strict diagnostics instead of printing them. `mysql.Writer.Statements` +performs the same preflight without opening a destination and returns structured +SQL statements suitable for execution without splitting on semicolons. + +Strict conversion currently covers **MySQL output**. It is not a general +losslessness check for other writers or a validator of arbitrary SQL fragments. +Warnings identify the object and omitted or changed feature. Inspect them before +using non-strict output; unsupported index types are omitted, while a partial +index's predicate and INCLUDE columns cannot be represented in MySQL. + +## MySQL capability matrix + +| Feature | Reader → model | Model → MySQL DDL | +| --- | --- | --- | +| Ordered columns, nullability, primary/composite keys | Preserved | Preserved | +| Unique/composite keys | Preserved, including backing indexes | Preserved once; no duplicate unique index | +| Foreign/composite keys and DELETE/UPDATE actions | Preserved | Added after all tables, allowing forward references/cycles | +| CHECK and NOT ENFORCED CHECK | Preserved | Preserved | +| BTREE, FULLTEXT, SPATIAL indexes | Preserved | Emitted; engine restrictions still apply | +| Expression/prefix/descending/invisible index parts | Preserved in `Index.KeyParts` and metadata | Preserved | +| HASH indexes | Preserved | Engine-dependent; incompatible engine produces a diagnostic | +| Integer unsignedness; mediumint; enum/set; binary/blob/text variants | Canonical type plus native declaration | Native declaration preserved | +| Length, numeric precision/scale, fractional seconds | Typed dimensions plus native declaration | Preserved; canonical dimensions also supported | +| Defaults, including SQL expressions, bit/binary literals and multiline text | Values plus native SQL spelling from SHOW CREATE | Preserved | +| Auto-increment and current next value | Preserved | Preserved | +| Stored/virtual generated columns, ON UPDATE, invisible columns | Preserved | Preserved | +| Engine, table/column collation, charset and comments | Preserved | Preserved | +| Views, triggers, routines, events, partitioned tables | Names recorded in `mysql_unsupported` | Diagnostic; bodies/partition definitions are not emitted | +| Nonempty table CREATE_OPTIONS; native parser/storage/engine attributes; spatial SRID | Recorded as unsupported attributes | Diagnostic; not emitted | +| Partial/INCLUDE indexes, unsupported index methods | Retained when supplied in the model | Diagnostic | +| Deferrable constraints, identity semantics, timestamptz, jsonb storage | Retained when supplied in the model | Diagnostic for unsupported/different semantics | +| Standalone sequences/enums/domains, ownership/permissions, scripts and dialect directives | Retained when supplied in the model | Diagnostic; not emitted | +| Data, grants, users, server/plugin configuration | Outside the reader's schema contract | Outside the writer's contract | + +Diagnostics depend on the objects visible to the connection account. They cover +known modeled/discovered features, not every engine/plugin extension. The server +still validates SQL expressions, engine rules, and references at execution time. +For example, source and target engines must support the constraints requested. + +## Preservation across intermediate formats + +| Path | Scope | +| --- | --- | +| MySQL → model → MySQL | Live-tested for the features above on MySQL 8.4 / InnoDB | +| MySQL → JSON → model → MySQL | Live-tested; preserves typed index parts and MySQL metadata | +| MySQL → YAML → model → MySQL | YAML serializes the same fields/metadata; covered by the live round-trip test | +| Other source/target formats | Their own mappings apply; MySQL-specific metadata and semantics are not certified to survive | + +Native declarations and defaults are stored in column metadata as `mysql_type` +and `mysql_default_sql`. Reader-generated signatures (`mysql_type_signature`, +`mysql_default_value`) prevent these spellings from silently overriding edited +types/dimensions/defaults. Editing canonical dimensions invalidates the native +type signature, produces a diagnostic, and uses the canonical mapping instead. +For a binary default, the logical default is its SQL hex/literal spelling so it +survives serialization without invalid UTF-8. + +Other metadata includes `mysql_charset`, `mysql_generation_storage`, +`mysql_default_expression`, `mysql_on_update`, `mysql_invisible`, table +`mysql_engine` / `mysql_collation` / `mysql_auto_increment`, index +`mysql_visible`, and check `mysql_enforced`. Unknown `mysql_*` metadata produces +a diagnostic. For a manually authored expression default, set +`mysql_default_expression: true`; otherwise strings are literal values. +`Index.KeyParts` is authoritative when present; `Index.Columns` remains the +compatibility representation for simple indexes. + +Raw native type declarations, default SQL, generation/check/index expressions, +and ON UPDATE expressions are executable SQL input. Lowercasing changes names +and structured index parts but leaves expressions untouched; update references +inside raw SQL yourself when using `--lowercase`. + +## Applying the output + +Create the destination database/schema first. Qualified names are preserved, +including foreign-key target schemas. Adjust them explicitly when restoring to +a different database. Output assumes standard MySQL backslash escaping; +`NO_BACKSLASH_ESCAPES` or `ANSI_QUOTES` on the source produces a conversion +diagnostic, and live execution refuses these target modes before DDL. File consumers must use the same +supported mode. Other SQL-mode incompatibilities surface as server errors. + +`CREATE TABLE IF NOT EXISTS` does not reconcile existing definitions. Foreign +keys use `ALTER TABLE ADD CONSTRAINT` and therefore repeated application can +fail on existing constraints. This is creation DDL, not a rerunnable migration. +Live execution stops on the first SQL error by default. Explicit +`ContinueOnError` attempts later statements and still returns an error if any +fail. MySQL DDL implicitly commits; `AtomicExecution` is rejected. PostgreSQL's +shared execution-report support is not yet implemented for this writer. + +## Verification + +```bash +./tests/dbtest/dbtest.sh test mysql +# Or use your own disposable database: +RELSPEC_TEST_MYSQL_CONN="$MYSQL_DSN" go test -mod=vendor ./pkg/readers/mysql ./pkg/writers/mysql +``` + +The live tests compare MySQL's `SHOW CREATE TABLE` definitions, normalizing only +temporary target-name suffixes and nonsemantic ordinary-index display order. +They re-read the target, exercise constraint enforcement/default values, and +verify unsupported objects produce warnings or strict failures without touching +an existing output file. Test fixtures are removed after each run. + +Metadata references: [COLUMNS](https://dev.mysql.com/doc/refman/8.4/en/information-schema-columns-table.html), +[STATISTICS](https://dev.mysql.com/doc/refman/8.4/en/information-schema-statistics-table.html), +[TABLE_CONSTRAINTS](https://dev.mysql.com/doc/refman/8.4/en/information-schema-table-constraints-table.html), +and [REFERENTIAL_CONSTRAINTS](https://dev.mysql.com/doc/refman/8.4/en/information-schema-referential-constraints-table.html). diff --git a/docs/audit_1.md b/docs/audit_1.md index f1d9c96..294e8e8 100644 --- a/docs/audit_1.md +++ b/docs/audit_1.md @@ -20,6 +20,11 @@ Implementation batches (2026-10-12): - [x] **Requested lowercase option:** `--lowercase` on convert/batch/merge/split/templ/job run, job `options.lowercase_identifiers`, and PostgreSQL `WriterOptions.LowercaseIdentifiers` normalize names and structured references on a copy, rejecting collisions. SQL expressions, defaults, comments, enum values, and data keep their case; callers must adjust references inside raw SQL themselves. - [x] **A08, SQL-aware substitution/execution:** embeds use one-pass literal substitution, preserving dollar characters and preventing replacement rescanning. Named parameters and embed directives respect SQL strings, quoted identifiers, nested comments, casts, and dollar bodies; duplicate/unused embed variables fail. MySQL executes structured generated statements without splitting multiline literals or stripping their contents. +- [x] **A06, nullable MySQL catalogs:** constraint references, functional-index columns/expressions, index prefix/direction fields, column dimensions/collations/generation metadata, and optional table auto-increment counters use nullable destinations. Queries qualify joined columns. Live primary/unique/composite/functional-key reads pass, including unsigned auto-increment values beyond signed 64-bit range. +- [x] **A07, scoped MySQL fidelity and diagnostics:** native types/default spellings and typed dimensions/index parts survive MySQL → model → JSON/YAML → DDL. Output emits foreign keys/actions after all tables, checks/enforcement, unique/ordinary/functional/prefix/descending/invisible/fulltext indexes, generated columns, comments/collations/charsets, ON UPDATE, invisible columns, and auto-increment counters. Known unsupported objects/attributes produce warnings or `--strict-conversion` / job `options.strict_conversion` / library `StrictConversion` failures before output or connection. See [the scoped capability matrix](MYSQL_FIDELITY.md); this covers new-table DDL on MySQL 8.4 / InnoDB, not all engines, MariaDB versions, other format mappings, or migrations. + +A06–A07 verification (2026-10-12): live MySQL 8.4.11 tests compare source/target `SHOW CREATE TABLE` definitions (normalizing temporary target suffixes and ordinary-index display order), re-read the target, and verify FK/check enforcement and binary/bit/multiline/backslash/default values. The suite covers composite/functional unique keys, reverse table dependency order, generated/check/index expressions containing SQL literals, precision, unsigned ranges, and multiple FKs to the same target. It exposed and fixed truncated binary defaults in `information_schema` by extracting native defaults from SQL-aware `SHOW CREATE TABLE` definitions. Native SHOW CREATE also avoids stale auto-increment statistics, and escaped catalog expressions are decoded before rendering. Unsupported views/triggers/partitioned tables produce warnings and strict failures; strict failure preserves existing files and happens before opening an output connection. CLI dry-run and per-job strict options are tested. The MySQL container test runner now selects reader/writer packages by default. `go test -mod=vendor -p 1 ./...` passed with `RELSPEC_TEST_MYSQL_CONN` configured; `go vet -mod=vendor ./...`, CLI strict/dry-run checks, shell syntax, and `git diff --check` passed. The MySQL container runner also passed using its schema-scoped account; its disposable server enables trigger-fixture creation after final-server readiness. PostgreSQL and SQL Server live variables were unset for this batch. Checks used `go1.27.1-X:nodwarf5 linux/amd64`; declared-minimum-toolchain coverage remains A13 work. Disposable MySQL instances were removed after verification. A08 now also has live MySQL coverage for embedded statement delimiters/default contents. + A05–A08 verification: unit regressions and live PostgreSQL 18 checks cover unusual names, comments/defaults, rerunnable file/statement output, diff migrations, sequences, primary-key replacement, audit functions/triggers, and both standard-string modes. MySQL statement preservation is unit-tested; a live MySQL instance was not used for A08. `go test -mod=vendor -p 1 ./...` passed with `RELSPEC_TEST_PG_CONN` set to the disposable instance; `go vet -mod=vendor ./...`, executable CLI help checks, and `git diff --check` passed. SQL Server tests were not configured for this batch. The disposable instance was removed after verification. Additional observation: a parallel live full-suite run hit a PostgreSQL reader catalog race (`could not open relation with OID`) while another package's test dropped schema objects. The writer safely fell back to full DDL and its preflight still stopped all mutations, but the existing test assumed no successful read-only preflights before failure. Running live packages serially passed. Reader resilience to concurrent catalog changes remains outside A05/A08; A10 specifically concerns cancellation/timeouts. @@ -155,6 +160,8 @@ PostgreSQL documents the distinct identifier, string, and dollar-quote rules in ### A06 — MySQL constraint reading scans nullable metadata into strings +**Repair status: addressed.** Nullable metadata scans and qualified queries are covered by live MySQL 8.4 round trips with primary, unique, composite FK, and functional-index definitions. + **P1 / High · Database · Code-confirmed; live MySQL not exercised** Evidence: `pkg/readers/mysql/reader.go:181–220`, specifically the `rs`, `rt`, and `rc` string variables passed to `rows.Scan`. @@ -169,6 +176,8 @@ Reference: [MySQL KEY_COLUMN_USAGE documentation](https://dev.mysql.com/doc/refm ### A07 — MySQL conversion silently loses constraints, indexes, and type detail +**Repair status: addressed within the [documented MySQL table-schema contract](MYSQL_FIDELITY.md).** Supported features survive live round trips through JSON/YAML; known unsupported modeled/discovered features produce diagnostics or strict conversion failures. MySQL output remains creation DDL for existing databases, with engine/version/SQL-mode limits stated in the capability matrix. + **P1 / High · Database · Reproduced omissions; additional losses code-confirmed** Evidence: `pkg/writers/mysql/writer.go:96–163`; `pkg/readers/mysql/reader.go:153–178`; `pkg/mariadb/datatypes.go`. diff --git a/pkg/jobs/jobs.go b/pkg/jobs/jobs.go index bcfeb50..3e533d0 100644 --- a/pkg/jobs/jobs.go +++ b/pkg/jobs/jobs.go @@ -277,6 +277,7 @@ func parseHumanSize(s string) (int64, error) { // Options carries the subset of command flags a job file may set. type Options struct { + StrictConversion bool `yaml:"strict_conversion"` LowercaseIdentifiers bool `yaml:"lowercase_identifiers"` FlattenSchema bool `yaml:"flatten_schema"` Schema string `yaml:"schema"` diff --git a/pkg/mariadb/datatypes.go b/pkg/mariadb/datatypes.go index aa5dbc9..1ecab6a 100644 --- a/pkg/mariadb/datatypes.go +++ b/pkg/mariadb/datatypes.go @@ -61,6 +61,7 @@ var MariaDBToCanonicalTypes = map[string]string{ // CanonicalToMariaDBTypes maps canonical types to MariaDB/MySQL types. var CanonicalToMariaDBTypes = map[string]string{ "bool": "TINYINT(1)", + "boolean": "TINYINT(1)", "int8": "TINYINT", "int16": "SMALLINT", "int": "INT", @@ -114,20 +115,26 @@ func NormalizeMariaDBType(baseType string) string { // ConvertMariaDBToCanonical converts a MariaDB/MySQL type name to the canonical type. // Strips dimension parameters and normalizes aliases. Defaults to "string". func ConvertMariaDBToCanonical(mariadbType string) string { - base := strings.ToLower(strings.TrimSpace(mariadbType)) - if idx := strings.Index(base, "("); idx >= 0 { - base = strings.TrimSpace(base[:idx]) + lower := strings.ToLower(strings.TrimSpace(mariadbType)) + base := lower + if idx := strings.IndexAny(base, "( "); idx >= 0 { + base = base[:idx] } - - if canonical, ok := MariaDBToCanonicalTypes[base]; ok { - return canonical - } - - // Prefix match for composite types (e.g., "unsigned bigint") - for key, canonical := range MariaDBToCanonicalTypes { - if strings.HasPrefix(base, key) { - return canonical + canonical, ok := MariaDBToCanonicalTypes[base] + if ok { + if strings.Contains(lower, "unsigned") || strings.Contains(lower, "zerofill") { + switch canonical { + case "int8": + return "uint8" + case "int16": + return "uint16" + case "int": + return "uint" + case "int64": + return "uint64" + } } + return canonical } return "string" @@ -145,12 +152,5 @@ func ConvertCanonicalToMariaDB(canonicalType string) string { return mariadbType } - // Prefix fallback - for canonical, mariadb := range CanonicalToMariaDBTypes { - if strings.HasPrefix(lower, canonical) { - return mariadb - } - } - return "VARCHAR(255)" } diff --git a/pkg/mariadb/datatypes_test.go b/pkg/mariadb/datatypes_test.go new file mode 100644 index 0000000..fa76508 --- /dev/null +++ b/pkg/mariadb/datatypes_test.go @@ -0,0 +1,13 @@ +package mariadb + +import "testing" + +func TestIntegerSignedness(t *testing.T) { + for _, tc := range []struct{ input, want string }{ + {"tinyint(3) unsigned", "uint8"}, {"smallint unsigned", "uint16"}, {"mediumint unsigned", "uint"}, {"int unsigned", "uint"}, {"bigint unsigned", "uint64"}, {"bigint zerofill", "uint64"}, {"bigint", "int64"}, {"varchar(32)", "string"}, + } { + if got := ConvertMariaDBToCanonical(tc.input); got != tc.want { + t.Errorf("%s: got %s want %s", tc.input, got, tc.want) + } + } +} diff --git a/pkg/models/models.go b/pkg/models/models.go index 8b183e1..57e0524 100644 --- a/pkg/models/models.go +++ b/pkg/models/models.go @@ -256,9 +256,19 @@ func (d *Column) SQLName() string { return strings.ToLower(d.Name) } +// IndexPart is an ordered MySQL index key part. Name and Expression are mutually +// exclusive; expressions are executable SQL and are not identifier-normalized. +type IndexPart struct { + Name string `json:"name,omitempty" yaml:"name,omitempty" xml:"name,omitempty"` + Expression string `json:"expression,omitempty" yaml:"expression,omitempty" xml:"expression,omitempty"` + Length int `json:"length,omitempty" yaml:"length,omitempty" xml:"length,omitempty"` + Descending bool `json:"descending,omitempty" yaml:"descending,omitempty" xml:"descending,omitempty"` +} + // Index represents a database index for optimizing query performance. // Indexes can be unique, partial, or include additional columns. type Index struct { + KeyParts []IndexPart `json:"key_parts,omitempty" yaml:"key_parts,omitempty" xml:"key_parts,omitempty"` Name string `json:"name" yaml:"name" xml:"name"` Description string `json:"description,omitempty" yaml:"description,omitempty" xml:"description,omitempty"` Table string `json:"table,omitempty" yaml:"table,omitempty" xml:"table,omitempty"` @@ -318,6 +328,7 @@ func (d *Relationship) SQLName() string { // Constraint represents a database constraint that enforces data integrity rules. // Constraints can be primary keys, foreign keys, unique constraints, check constraints, or not-null constraints. type Constraint struct { + Metadata map[string]any `json:"metadata,omitempty" yaml:"metadata,omitempty" xml:"-"` Name string `json:"name" yaml:"name" xml:"name"` Type ConstraintType `json:"type" yaml:"type" xml:"type"` Columns []string `json:"columns" yaml:"columns" xml:"columns"` diff --git a/pkg/readers/mysql/defaults.go b/pkg/readers/mysql/defaults.go new file mode 100644 index 0000000..2f0954d --- /dev/null +++ b/pkg/readers/mysql/defaults.go @@ -0,0 +1,194 @@ +package mysql + +import ( + "regexp" + "strings" +) + +// columnDefaults extracts native DEFAULT expressions from SHOW CREATE TABLE. +// MySQL can truncate binary defaults in information_schema (for example X'00FF' +// appears as just "0x"). SHOW CREATE is authoritative for their SQL spelling. +// Separators and whitespace inside quoted strings, identifiers and nested +// expressions are never treated as definition/default boundaries. +func columnDefaults(create string) map[string]string { + out := map[string]string{} + depth, start := 0, 0 + var quoted byte + definitions := []string{} + for i := 0; i < len(create); i++ { + ch := create[i] + if quoted != 0 { + if ch == '\\' && quoted != '`' && i+1 < len(create) { + i++ + continue + } + if ch == quoted { + if i+1 < len(create) && create[i+1] == quoted { + i++ + } else { + quoted = 0 + } + } + continue + } + if ch == '\'' || ch == '"' || ch == '`' { + quoted = ch + continue + } + switch ch { + case '(': + depth++ + if depth == 1 { + start = i + 1 + } + case ',': + if depth == 1 { + definitions = append(definitions, create[start:i]) + start = i + 1 + } + case ')': + depth-- + if depth == 0 { + definitions = append(definitions, create[start:i]) + break + } + } + if ch == ')' && depth == 0 { + break + } + } + for _, def := range definitions { + def = strings.TrimSpace(def) + if len(def) == 0 || def[0] != '`' { + continue + } + var name strings.Builder + end := 1 + for end < len(def) { + if def[end] == '`' { + if end+1 < len(def) && def[end+1] == '`' { + name.WriteByte('`') + end += 2 + continue + } + end++ + break + } + name.WriteByte(def[end]) + end++ + } + // Consume full tokens so DEFAULT inside a generated expression or literal + // cannot masquerade as a column default. + for end < len(def) { + for end < len(def) && isSpace(def[end]) { + end++ + } + begin := end + end = tokenEnd(def, end) + if end == begin { + break + } + if strings.EqualFold(def[begin:end], "DEFAULT") { + for end < len(def) && isSpace(def[end]) { + end++ + } + finish := tokenEnd(def, end) + out[name.String()] = def[end:finish] + break + } + } + } + return out +} +func isSpace(b byte) bool { return b == ' ' || b == '\n' || b == '\r' || b == '\t' } +func tokenEnd(s string, start int) int { + depth := 0 + var quoted byte + for i := start; i < len(s); i++ { + ch := s[i] + if quoted != 0 { + if ch == '\\' && quoted != '`' && i+1 < len(s) { + i++ + continue + } + if ch == quoted { + if i+1 < len(s) && s[i+1] == quoted { + i++ + } else { + quoted = 0 + } + } + continue + } + if ch == '\'' || ch == '"' || ch == '`' { + quoted = ch + continue + } + if ch == '(' { + depth++ + } + if ch == ')' { + depth-- + } + if depth == 0 && isSpace(ch) { + return i + } + } + return len(s) +} + +// SHOW CREATE has native options not exposed by STATISTICS/COLUMNS. Ignore +// quoted identifiers and values when detecting options outside our contract. +func unquotedSQL(create string) string { + var code strings.Builder + var quoted byte + for i := 0; i < len(create); i++ { + ch := create[i] + if quoted != 0 { + if ch == '\\' && quoted != '`' && i+1 < len(create) { + i++ + continue + } + if ch == quoted { + if i+1 < len(create) && create[i+1] == quoted { + i++ + } else { + quoted = 0 + code.WriteByte(' ') + } + } + continue + } + if ch == '\'' || ch == '"' || ch == '`' { + quoted = ch + code.WriteByte(' ') + continue + } + code.WriteByte(ch) + } + return code.String() +} + +var autoIncrementOption = regexp.MustCompile(`(?i)\bAUTO_INCREMENT\s*=\s*([0-9]+)`) + +func autoIncrementValue(create string) string { + if match := autoIncrementOption.FindStringSubmatch(unquotedSQL(create)); len(match) > 1 { + return match[1] + } + return "1" +} +func unsupportedNativeOptions(create string) []string { + words := strings.FieldsFunc(strings.ToUpper(unquotedSQL(create)), func(r rune) bool { return !(r >= 'A' && r <= 'Z' || r == '_') }) + var out []string + for i, w := range words { + switch w { + case "ENGINE_ATTRIBUTE", "SECONDARY_ENGINE_ATTRIBUTE", "KEY_BLOCK_SIZE", "COLUMN_FORMAT", "STORAGE": + out = append(out, w) + case "PARSER": + if i > 0 && words[i-1] == "WITH" { + out = append(out, "WITH PARSER") + } + } + } + return out +} diff --git a/pkg/readers/mysql/expressions_test.go b/pkg/readers/mysql/expressions_test.go new file mode 100644 index 0000000..31387e8 --- /dev/null +++ b/pkg/readers/mysql/expressions_test.go @@ -0,0 +1,54 @@ +package mysql + +import "testing" + +func TestUnescapeExpression(t *testing.T) { + for _, tc := range []struct{ input, want string }{ + {`_utf8mb4\'one\\ntwo\'`, `_utf8mb4'one\ntwo'`}, + {`concat(\'don\\\'t\',\' change\')`, `concat('don\'t',' change')`}, + {`(uuid())`, `(uuid())`}, + } { + if got := unescapeExpression(tc.input); got != tc.want { + t.Errorf("%q: got %q want %q", tc.input, got, tc.want) + } + } +} + +func TestNativeColumnDefaults(t *testing.T) { + create := "CREATE TABLE `odd(table` (\n" + + " `a,b` varbinary(8) DEFAULT 0x0027FF,\n" + + " `tick``name` varchar(30) DEFAULT 'don''t, (split)',\n" + + " `expr` varchar(30) DEFAULT (concat('DEFAULT, ', (uuid()))),\n" + + " `generated` varchar(30) GENERATED ALWAYS AS (concat('DEFAULT',`expr`)) VIRTUAL,\n" + + " `new\nline` int DEFAULT 3,\n" + + " PRIMARY KEY (`a,b`)\n) ENGINE=InnoDB" + got := columnDefaults(create) + want := map[string]string{"a,b": "0x0027FF", "tick`name": "'don''t, (split)'", "expr": "(concat('DEFAULT, ', (uuid())))", "new\nline": "3"} + if len(got) != len(want) { + t.Fatalf("defaults: %v", got) + } + for name, value := range want { + if got[name] != value { + t.Errorf("%s: got %q want %q", name, got[name], value) + } + } +} + +func TestNativeOptionsIgnoreQuotedValues(t *testing.T) { + create := "CREATE TABLE `WITH PARSER` (`STORAGE` int COMMENT 'KEY_BLOCK_SIZE', KEY `ENGINE_ATTRIBUTE` (`STORAGE`) WITH PARSER `ngram`) ENGINE=InnoDB" + options := unsupportedNativeOptions(create) + if len(options) != 1 || options[0] != "WITH PARSER" { + t.Fatalf("native options: %v", options) + } +} + +func TestNativeAutoIncrementIgnoresQuotedValues(t *testing.T) { + for _, tc := range []struct{ create, want string }{ + {"CREATE TABLE `x` (`id` bigint AUTO_INCREMENT) ENGINE=InnoDB AUTO_INCREMENT=18446744073709551615", "18446744073709551615"}, + {"CREATE TABLE `x` (`id` bigint AUTO_INCREMENT) COMMENT='AUTO_INCREMENT=12'", "1"}, + } { + if got := autoIncrementValue(tc.create); got != tc.want { + t.Errorf("got %s want %s", got, tc.want) + } + } +} diff --git a/pkg/readers/mysql/reader.go b/pkg/readers/mysql/reader.go index 1112810..c3e60be 100644 --- a/pkg/readers/mysql/reader.go +++ b/pkg/readers/mysql/reader.go @@ -4,9 +4,10 @@ import ( "context" "database/sql" "fmt" + "regexp" "strings" - _ "github.com/go-sql-driver/mysql" + mysqlDriver "github.com/go-sql-driver/mysql" "git.warky.dev/wdevs/relspecgo/pkg/mariadb" "git.warky.dev/wdevs/relspecgo/pkg/models" @@ -31,12 +32,13 @@ func (r *Reader) ReadDatabase() (*models.Database, error) { return nil, fmt.Errorf("failed to connect: %w", err) } defer r.close() - var name, version string - if err := r.db.QueryRowContext(r.ctx, "SELECT DATABASE()").Scan(&name); err != nil { + var name, version, sqlMode string + if err := r.db.QueryRowContext(r.ctx, "SELECT DATABASE(), @@SESSION.sql_mode").Scan(&name, &sqlMode); err != nil { return nil, fmt.Errorf("failed to get database name: %w", err) } _ = r.db.QueryRowContext(r.ctx, "SELECT VERSION()").Scan(&version) db := models.InitDatabase(name) + db.Metadata["mysql_sql_mode"] = sqlMode db.DatabaseType, db.SourceFormat, db.DatabaseVersion = models.MySQLDatabaseType, "mysql", version schemas, err := r.querySchemas(name) if err != nil { @@ -48,8 +50,10 @@ func (r *Reader) ReadDatabase() (*models.Database, error) { return nil, err } schema.Tables = tables + schema.Metadata["mysql_sql_mode"] = sqlMode for _, table := range tables { - table.Columns, err = r.queryColumns(schema.Name, table.Name) + table.Metadata["mysql_sql_mode"] = sqlMode + table.Columns, err = r.queryColumns(schema.Name, table) if err != nil { return nil, err } @@ -62,6 +66,15 @@ func (r *Reader) ReadDatabase() (*models.Database, error) { return nil, err } table.RefSchema = schema + for _, c := range table.Constraints { + if c.Type == models.PrimaryKeyConstraint { + for _, n := range c.Columns { + if col := table.Columns[n]; col != nil { + col.IsPrimaryKey = true + } + } + } + } for _, c := range table.Constraints { if c.Type == models.ForeignKeyConstraint { r.deriveRelationship(table, c) @@ -71,6 +84,9 @@ func (r *Reader) ReadDatabase() (*models.Database, error) { schema.RefDatabase = db db.Schemas = append(db.Schemas, schema) } + if err := r.readUnsupported(db, name); err != nil { + return nil, err + } return db, nil } @@ -97,7 +113,17 @@ func (r *Reader) ReadTable() (*models.Table, error) { } func (r *Reader) connect() error { - db, err := sql.Open("mysql", r.options.ConnectionString) + cfg, err := mysqlDriver.ParseDSN(r.options.ConnectionString) + if err != nil { + return err + } + if cfg.Params == nil { + cfg.Params = map[string]string{} + } + // SHOW CREATE parsing requires quoted column identifiers, including when the + // caller's metadata connection would otherwise disable show-create quoting. + cfg.Params["sql_quote_show_create"] = "1" + db, err := sql.Open("mysql", cfg.FormatDSN()) if err != nil { return err } @@ -134,61 +160,147 @@ func (r *Reader) querySchemas(current string) ([]*models.Schema, error) { } func (r *Reader) queryTables(schema string) ([]*models.Table, error) { - rows, err := r.db.QueryContext(r.ctx, "SELECT TABLE_NAME FROM information_schema.TABLES WHERE TABLE_SCHEMA = ? AND TABLE_TYPE = 'BASE TABLE' ORDER BY TABLE_NAME", schema) + rows, err := r.db.QueryContext(r.ctx, "SELECT TABLE_NAME, ENGINE, TABLE_COLLATION, TABLE_COMMENT, CREATE_OPTIONS, AUTO_INCREMENT FROM information_schema.TABLES WHERE TABLE_SCHEMA = ? AND TABLE_TYPE = 'BASE TABLE' ORDER BY TABLE_NAME", schema) if err != nil { return nil, err } defer rows.Close() var out []*models.Table for rows.Next() { - var n string - if err := rows.Scan(&n); err != nil { + var n, engine, collation, comment, options string + var auto sql.NullString + if err := rows.Scan(&n, &engine, &collation, &comment, &options, &auto); err != nil { return nil, err } - out = append(out, models.InitTable(n, schema)) + t := models.InitTable(n, schema) + t.Comment = comment + t.Metadata["mysql_engine"], t.Metadata["mysql_collation"], t.Metadata["mysql_create_options"] = engine, collation, options + if auto.Valid { + t.Metadata["mysql_auto_increment"] = auto.String + } + out = append(out, t) } return out, rows.Err() } -func (r *Reader) queryColumns(schema, table string) (map[string]*models.Column, error) { - rows, err := r.db.QueryContext(r.ctx, `SELECT COLUMN_NAME, COLUMN_TYPE, IS_NULLABLE, COLUMN_DEFAULT, ORDINAL_POSITION, EXTRA, COLUMN_COMMENT FROM information_schema.COLUMNS WHERE TABLE_SCHEMA = ? AND TABLE_NAME = ? ORDER BY ORDINAL_POSITION`, schema, table) +var onUpdateExtra = regexp.MustCompile(`on update current_timestamp(?:\(\d*\))?`) + +func (r *Reader) queryColumns(schema string, table *models.Table) (map[string]*models.Column, error) { + rows, err := r.db.QueryContext(r.ctx, `SELECT COLUMN_NAME, COLUMN_TYPE, IS_NULLABLE, COLUMN_DEFAULT, ORDINAL_POSITION, EXTRA, COLUMN_COMMENT, + CHARACTER_MAXIMUM_LENGTH, NUMERIC_PRECISION, NUMERIC_SCALE, DATETIME_PRECISION, COLLATION_NAME, CHARACTER_SET_NAME, GENERATION_EXPRESSION, SRS_ID + FROM information_schema.COLUMNS WHERE TABLE_SCHEMA=? AND TABLE_NAME=? ORDER BY ORDINAL_POSITION`, schema, table.Name) if err != nil { - return nil, err + return nil, fmt.Errorf("columns %s.%s: %w", schema, table.Name, err) } defer rows.Close() out := map[string]*models.Column{} for rows.Next() { var name, typ, nullable, extra, comment string - var def sql.NullString + var def, collation, charset, generation sql.NullString + var length, precision, scale, datetime, srid sql.NullInt64 var pos int - if err := rows.Scan(&name, &typ, &nullable, &def, &pos, &extra, &comment); err != nil { + if err := rows.Scan(&name, &typ, &nullable, &def, &pos, &extra, &comment, &length, &precision, &scale, &datetime, &collation, &charset, &generation, &srid); err != nil { return nil, err } - c := models.InitColumn(name, table, schema) - c.Type = r.mapDataType(typ) - c.NotNull = strings.EqualFold(nullable, "NO") - c.Sequence = uint(pos) - c.Comment = comment + c := models.InitColumn(name, table.Name, schema) + c.Type, c.NotNull, c.Sequence, c.Comment = r.mapDataType(typ), nullable == "NO", uint(pos), comment + // The native declaration preserves mediumint, enum/set, binary/blob sizes, + // unsigned/zerofill and timestamp versus datetime. Canonical dimensions remain editable. + c.Length, c.Precision, c.Scale = int(length.Int64), int(precision.Int64), int(scale.Int64) + if datetime.Valid { + c.Precision = int(datetime.Int64) + } + c.Collation = collation.String + c.Metadata["mysql_type"] = typ + c.Metadata["mysql_type_signature"] = typeSignature(c) + c.Metadata["mysql_charset"] = charset.String + c.Metadata["mysql_datetime_precision"] = datetime.Int64 + c.Metadata["mysql_default_expression"] = strings.Contains(extra, "DEFAULT_GENERATED") || strings.HasPrefix(strings.ToLower(def.String), "current_timestamp") && (c.Type == "timestamp") if def.Valid { c.Default = def.String + if strings.Contains(extra, "DEFAULT_GENERATED") { + c.Default = unescapeExpression(def.String) + } + if strings.HasPrefix(typ, "bit(") && def.Valid { + c.Metadata["mysql_default_expression"] = true + } + } + c.AutoIncrement = strings.Contains(extra, "auto_increment") + c.Generated, c.GenerationExpression = generation.String != "", unescapeExpression(generation.String) + if strings.Contains(extra, "STORED GENERATED") { + c.Metadata["mysql_generation_storage"] = "STORED" + } else if c.Generated { + c.Metadata["mysql_generation_storage"] = "VIRTUAL" + } + c.Metadata["mysql_invisible"] = strings.Contains(extra, "INVISIBLE") + remaining := onUpdateExtra.ReplaceAllString(strings.ToLower(extra), "") + for _, known := range []string{"auto_increment", "default_generated", "stored generated", "virtual generated", "invisible"} { + remaining = strings.ReplaceAll(remaining, known, "") + } + if remaining = strings.TrimSpace(remaining); remaining != "" { + c.Metadata["mysql_unsupported_extra"] = remaining + } + if i := strings.Index(strings.ToLower(extra), "on update "); i >= 0 { + c.Metadata["mysql_on_update"] = strings.Fields(extra[i+10:])[0] + } + if srid.Valid { + c.Metadata["mysql_srs_id"] = srid.Int64 } - c.AutoIncrement = strings.Contains(strings.ToLower(extra), "auto_increment") out[name] = c } - return out, rows.Err() + if err := rows.Err(); err != nil { + return nil, err + } + var tableName, create string + qualified := "`" + strings.ReplaceAll(schema, "`", "``") + "`.`" + strings.ReplaceAll(table.Name, "`", "``") + "`" + if err := r.db.QueryRowContext(r.ctx, "SHOW CREATE TABLE "+qualified).Scan(&tableName, &create); err != nil { + return nil, fmt.Errorf("native defaults %s.%s: %w", schema, table.Name, err) + } + // information_schema AUTO_INCREMENT may be stale because of statistics caching. + for _, c := range out { + if c.AutoIncrement { + table.Metadata["mysql_auto_increment"] = autoIncrementValue(create) + break + } + } + if options := unsupportedNativeOptions(create); len(options) > 0 { + table.Metadata["mysql_unsupported_options"] = strings.Join(options, ", ") + } + for name, sql := range columnDefaults(create) { + c := out[name] + if c == nil || c.Default == nil { + continue + } + if c.Type == "bytea" { + c.Default = sql + } + c.Metadata["mysql_default_sql"] = sql + c.Metadata["mysql_default_value"] = fmt.Sprint(c.Default) + } + return out, nil +} + +func typeSignature(c *models.Column) string { + return fmt.Sprintf("%s/%d/%d/%d", c.Type, c.Length, c.Precision, c.Scale) } func (r *Reader) queryConstraints(schema, table string) (map[string]*models.Constraint, error) { - rows, err := r.db.QueryContext(r.ctx, `SELECT CONSTRAINT_NAME, CONSTRAINT_TYPE, COLUMN_NAME, REFERENCED_TABLE_SCHEMA, REFERENCED_TABLE_NAME, REFERENCED_COLUMN_NAME, ORDINAL_POSITION FROM information_schema.KEY_COLUMN_USAGE k JOIN information_schema.TABLE_CONSTRAINTS t USING (CONSTRAINT_SCHEMA, TABLE_NAME, CONSTRAINT_NAME) WHERE k.TABLE_SCHEMA=? AND k.TABLE_NAME=? ORDER BY CONSTRAINT_NAME, ORDINAL_POSITION`, schema, table) + rows, err := r.db.QueryContext(r.ctx, `SELECT k.CONSTRAINT_NAME, t.CONSTRAINT_TYPE, k.COLUMN_NAME, + k.REFERENCED_TABLE_SCHEMA, k.REFERENCED_TABLE_NAME, k.REFERENCED_COLUMN_NAME, k.ORDINAL_POSITION, + rc.DELETE_RULE, rc.UPDATE_RULE FROM information_schema.KEY_COLUMN_USAGE k + JOIN information_schema.TABLE_CONSTRAINTS t USING (CONSTRAINT_SCHEMA,TABLE_NAME,CONSTRAINT_NAME) + LEFT JOIN information_schema.REFERENTIAL_CONSTRAINTS rc ON rc.CONSTRAINT_SCHEMA=k.CONSTRAINT_SCHEMA AND rc.TABLE_NAME=k.TABLE_NAME AND rc.CONSTRAINT_NAME=k.CONSTRAINT_NAME + WHERE k.TABLE_SCHEMA=? AND k.TABLE_NAME=? ORDER BY k.CONSTRAINT_NAME,k.ORDINAL_POSITION`, schema, table) if err != nil { - return nil, err + return nil, fmt.Errorf("constraints %s.%s: %w", schema, table, err) } - defer rows.Close() out := map[string]*models.Constraint{} for rows.Next() { - var name, typ, col, rs, rt, rc string + var name, typ string + var col, rs, rt, rc, del, upd sql.NullString var pos int - if err := rows.Scan(&name, &typ, &col, &rs, &rt, &rc, &pos); err != nil { + if err := rows.Scan(&name, &typ, &col, &rs, &rt, &rc, &pos, &del, &upd); err != nil { + rows.Close() return nil, err } c := out[name] @@ -201,31 +313,54 @@ func (r *Reader) queryConstraints(schema, table string) (map[string]*models.Cons ct = models.ForeignKeyConstraint } c = models.InitConstraint(name, ct) - c.Schema = schema - c.Table = table - c.ReferencedSchema = rs - c.ReferencedTable = rt + c.Schema, c.Table = schema, table + c.ReferencedSchema, c.ReferencedTable = rs.String, rt.String + c.OnDelete, c.OnUpdate = del.String, upd.String out[name] = c } - c.Columns = append(c.Columns, col) - if rc != "" { - c.ReferencedColumns = append(c.ReferencedColumns, rc) + if col.Valid { + c.Columns = append(c.Columns, col.String) } + if rc.Valid { + c.ReferencedColumns = append(c.ReferencedColumns, rc.String) + } + } + err = rows.Err() + rows.Close() + if err != nil { + return nil, err + } + rows, err = r.db.QueryContext(r.ctx, `SELECT t.CONSTRAINT_NAME,c.CHECK_CLAUSE,t.ENFORCED FROM information_schema.TABLE_CONSTRAINTS t JOIN information_schema.CHECK_CONSTRAINTS c USING (CONSTRAINT_SCHEMA,CONSTRAINT_NAME) WHERE t.TABLE_SCHEMA=? AND t.TABLE_NAME=? AND t.CONSTRAINT_TYPE='CHECK' ORDER BY t.CONSTRAINT_NAME`, schema, table) + if err != nil { + return nil, fmt.Errorf("checks %s.%s (requires MySQL 8.0.16+): %w", schema, table, err) + } + defer rows.Close() + for rows.Next() { + var name, expr, enforced string + if err := rows.Scan(&name, &expr, &enforced); err != nil { + return nil, err + } + c := models.InitConstraint(name, models.CheckConstraint) + c.Schema, c.Table, c.Expression = schema, table, unescapeExpression(expr) + c.Metadata = map[string]any{"mysql_enforced": enforced == "YES"} + out[name] = c } return out, rows.Err() } func (r *Reader) queryIndexes(schema, table string) (map[string]*models.Index, error) { - rows, err := r.db.QueryContext(r.ctx, `SELECT INDEX_NAME, NON_UNIQUE, COLUMN_NAME, SEQ_IN_INDEX, INDEX_TYPE FROM information_schema.STATISTICS WHERE TABLE_SCHEMA=? AND TABLE_NAME=? AND INDEX_NAME <> 'PRIMARY' ORDER BY INDEX_NAME, SEQ_IN_INDEX`, schema, table) + rows, err := r.db.QueryContext(r.ctx, `SELECT INDEX_NAME, NON_UNIQUE, COLUMN_NAME, SEQ_IN_INDEX, INDEX_TYPE, EXPRESSION, SUB_PART, COLLATION, IS_VISIBLE, INDEX_COMMENT FROM information_schema.STATISTICS WHERE TABLE_SCHEMA=? AND TABLE_NAME=? ORDER BY INDEX_NAME,SEQ_IN_INDEX`, schema, table) if err != nil { - return nil, err + return nil, fmt.Errorf("indexes %s.%s (requires MySQL 8.0.13+): %w", schema, table, err) } defer rows.Close() out := map[string]*models.Index{} for rows.Next() { - var name, col, typ string + var name, typ, visible, comment string + var col, expr, direction sql.NullString + var prefix sql.NullInt64 var non, seq int - if err := rows.Scan(&name, &non, &col, &seq, &typ); err != nil { + if err := rows.Scan(&name, &non, &col, &seq, &typ, &expr, &prefix, &direction, &visible, &comment); err != nil { return nil, err } i := out[name] @@ -233,22 +368,96 @@ func (r *Reader) queryIndexes(schema, table string) (map[string]*models.Index, e i = models.InitIndex(name, table, schema) i.Unique = non == 0 i.Type = strings.ToLower(typ) + i.Comment = comment + i.Metadata["mysql_visible"] = visible == "YES" out[name] = i } - i.Columns = append(i.Columns, col) + i.Columns = append(i.Columns, col.String) + i.KeyParts = append(i.KeyParts, models.IndexPart{Name: col.String, Expression: unescapeExpression(expr.String), Length: int(prefix.Int64), Descending: direction.String == "D"}) } return out, rows.Err() } +// Record objects outside the supported table-schema contract rather than silently +// discarding them. Writers can warn or reject the conversion before any output. +func (r *Reader) readUnsupported(db *models.Database, schema string) error { + queries := []struct{ kind, sql string }{ + {"view", `SELECT TABLE_NAME FROM information_schema.VIEWS WHERE TABLE_SCHEMA=?`}, + {"trigger", `SELECT TRIGGER_NAME FROM information_schema.TRIGGERS WHERE TRIGGER_SCHEMA=?`}, + {"routine", `SELECT ROUTINE_NAME FROM information_schema.ROUTINES WHERE ROUTINE_SCHEMA=?`}, + {"event", `SELECT EVENT_NAME FROM information_schema.EVENTS WHERE EVENT_SCHEMA=?`}, + {"partitioned table", `SELECT DISTINCT TABLE_NAME FROM information_schema.PARTITIONS WHERE TABLE_SCHEMA=? AND PARTITION_NAME IS NOT NULL`}, + } + var unsupported []string + for _, q := range queries { + rows, err := r.db.QueryContext(r.ctx, q.sql, schema) + if err != nil { + return fmt.Errorf("discover %s: %w", q.kind, err) + } + for rows.Next() { + var name string + if err := rows.Scan(&name); err != nil { + rows.Close() + return err + } + unsupported = append(unsupported, q.kind+" "+schema+"."+name) + } + err = rows.Err() + rows.Close() + if err != nil { + return err + } + } + if len(unsupported) > 0 { + db.Metadata["mysql_unsupported"] = unsupported + for _, s := range db.Schemas { + s.Metadata["mysql_unsupported"] = unsupported + } + if r.options.Progress != nil { + for _, item := range unsupported { + r.options.Progress("warning: unsupported MySQL " + item) + } + } + } + return nil +} + func (r *Reader) deriveRelationship(t *models.Table, c *models.Constraint) { - n := fmt.Sprintf("%s_to_%s", t.Name, c.ReferencedTable) + n := fmt.Sprintf("%s_%s", t.Name, c.Name) rel := models.InitRelationship(n, models.OneToMany) - rel.FromTable = t.Name - rel.FromSchema = t.Schema + rel.FromTable, rel.FromSchema = t.Name, t.Schema rel.FromColumns = append([]string(nil), c.Columns...) - rel.ToTable = c.ReferencedTable - rel.ToSchema = c.ReferencedSchema + rel.ToTable, rel.ToSchema = c.ReferencedTable, c.ReferencedSchema rel.ToColumns = append([]string(nil), c.ReferencedColumns...) rel.ForeignKey = c.Name t.Relationships[n] = rel } + +// MySQL information_schema escapes SQL expressions by an additional layer +// (including quote delimiters). This covers defaults, generated columns, checks, +// and functional indexes. Literal defaults are already values and bypass this. +func unescapeExpression(s string) string { + var out strings.Builder + for i := 0; i < len(s); i++ { + if s[i] == '\\' && i+1 < len(s) { + i++ + switch s[i] { + case 'n': + out.WriteByte('\n') + case 'r': + out.WriteByte('\r') + case 't': + out.WriteByte('\t') + case '0': + out.WriteByte(0) + case 'Z': + out.WriteByte(26) + default: + out.WriteByte(s[i]) + } + } else { + out.WriteByte(s[i]) + } + } + return out.String() +} diff --git a/pkg/readers/mysql/reader_test.go b/pkg/readers/mysql/reader_test.go index db0c544..5856b2b 100644 --- a/pkg/readers/mysql/reader_test.go +++ b/pkg/readers/mysql/reader_test.go @@ -3,12 +3,13 @@ package mysql import ( "testing" + "git.warky.dev/wdevs/relspecgo/pkg/models" "git.warky.dev/wdevs/relspecgo/pkg/readers" ) func TestReaderMapDataType(t *testing.T) { r := NewReader(&readers.ReaderOptions{}) - for _, tc := range []struct{ input, want string }{{"varchar(64)", "string"}, {"bigint unsigned", "int64"}, {"datetime", "timestamp"}, {"json", "json"}} { + for _, tc := range []struct{ input, want string }{{"varchar(64)", "string"}, {"bigint unsigned", "uint64"}, {"datetime", "timestamp"}, {"json", "json"}} { if got := r.mapDataType(tc.input); got != tc.want { t.Errorf("mapDataType(%q) = %q, want %q", tc.input, got, tc.want) } @@ -20,3 +21,17 @@ func TestReaderRequiresConnectionString(t *testing.T) { t.Fatal("expected missing connection string error") } } + +func TestRelationshipNamesCannotCollapseDistinctForeignKeys(t *testing.T) { + table := models.InitTable("t", "app") + reader := NewReader(&readers.ReaderOptions{}) + for _, c := range []*models.Constraint{ + {Name: "a_to_b", ReferencedTable: "c"}, + {Name: "a", ReferencedTable: "b_to_c"}, + } { + reader.deriveRelationship(table, c) + } + if len(table.Relationships) != 2 { + t.Fatalf("distinct foreign keys collapsed: %+v", table.Relationships) + } +} diff --git a/pkg/writers/mysql/conversion_test.go b/pkg/writers/mysql/conversion_test.go new file mode 100644 index 0000000..af35c83 --- /dev/null +++ b/pkg/writers/mysql/conversion_test.go @@ -0,0 +1,145 @@ +package mysql + +import ( + "os" + "path/filepath" + "strings" + "testing" + + "git.warky.dev/wdevs/relspecgo/pkg/models" + "git.warky.dev/wdevs/relspecgo/pkg/transform" + "git.warky.dev/wdevs/relspecgo/pkg/writers" +) + +func TestStrictConversionPreflightsAllObjects(t *testing.T) { + db := shopDB() + unsupported := models.InitTable("unsupported", "shop") + col := models.InitColumn("opaque", "unsupported", "shop") + col.Type = "unknown_type" + unsupported.Columns[col.Name] = col + index := models.InitIndex("partial", "unsupported", "shop") + index.Columns = []string{col.Name} + index.Where = "opaque IS NOT NULL" + index.Include = []string{"other"} + unsupported.Indexes[index.Name] = index + db.Schemas[0].Tables = append(db.Schemas[0].Tables, unsupported) + out := filepath.Join(t.TempDir(), "existing.sql") + if err := os.WriteFile(out, []byte("keep existing contents"), 0600); err != nil { + t.Fatal(err) + } + opts := &writers.WriterOptions{StrictConversion: true, OutputPath: out} + err := NewWriter(opts).WriteDatabase(db) + for _, want := range []string{"lossy MySQL conversion rejected", "unknown type", "partial predicate", "INCLUDE"} { + if err == nil || !strings.Contains(err.Error(), want) { + t.Fatalf("expected %s: %v", want, err) + } + } + bytes, err := os.ReadFile(out) + if err != nil || string(bytes) != "keep existing contents" { + t.Fatalf("destination changed: %q %v", bytes, err) + } + opts.Metadata = map[string]any{"connection_string": "invalid DSN"} + if err := NewWriter(opts).WriteDatabase(db); err == nil || !strings.Contains(err.Error(), "lossy MySQL") { + t.Fatalf("connected before strict validation: %v", err) + } + opts.StrictConversion = false + opts.Metadata = nil + var warnings []string + opts.Warning = func(s string) { warnings = append(warnings, s) } + if err := NewWriter(opts).WriteDatabase(db); err != nil { + t.Fatal(err) + } + if len(warnings) != 3 { + t.Fatalf("warnings: %v", warnings) + } +} + +func TestUnsupportedModelFeaturesDiagnosed(t *testing.T) { + db := shopDB() + s := db.Schemas[0] + table := s.Tables[0] + s.Metadata["mysql_unsupported"] = []any{"trigger shop.audit", "partitioned table shop.archive"} + c := table.Columns["id"] + c.Type = "timestamptz" + c.Identity = true + c.Metadata["mysql_srs_id"] = 4326 + check := models.InitConstraint("deferred", models.CheckConstraint) + check.Expression = "id > 0" + yes := true + check.Deferrable = &yes + table.Constraints[check.Name] = check + i := models.InitIndex("gin", "users", "shop") + i.Type = "gin" + i.Columns = []string{"id"} + table.Indexes[i.Name] = i + _, err := NewWriter(&writers.WriterOptions{StrictConversion: true}).Statements(db) + for _, want := range []string{"trigger shop.audit", "partitioned table", "time zone", "identity", "mysql_srs_id", "deferrable", "unsupported index type gin"} { + if err == nil || !strings.Contains(err.Error(), want) { + t.Errorf("missing diagnostic %q: %v", want, err) + } + } +} + +func TestCanonicalDimensionsAndNativeEdit(t *testing.T) { + db := shopDB() + c := db.Schemas[0].Tables[0].Columns["age"] + c.Type = "decimal" + c.Precision = 12 + c.Scale = 4 + if got := columnType(c); got != "DECIMAL(12,4)" { + t.Fatal(got) + } + c.Type = "uint64" + c.Precision = 0 + c.Scale = 0 + if got := columnType(c); got != "BIGINT UNSIGNED" { + t.Fatal(got) + } + c.Metadata["mysql_type"] = "bigint unsigned" + c.Metadata["mysql_type_signature"] = signature(c) + c.Type = "int64" + _, err := NewWriter(&writers.WriterOptions{StrictConversion: true}).Statements(db) + if err == nil || !strings.Contains(err.Error(), "override native declaration") { + t.Fatalf("native metadata silently overrode edit: %v", err) + } + if got := columnType(c); got != "BIGINT" { + t.Fatal(got) + } +} + +func TestLowercaseIndexPartsPreserveSQL(t *testing.T) { + db := shopDB() + table := db.Schemas[0].Tables[0] + c := models.InitColumn("Mixed", "users", "shop") + c.Type = "string" + table.Columns[c.Name] = c + i := models.InitIndex("MixedIndex", "users", "shop") + i.KeyParts = []models.IndexPart{{Name: "Mixed", Length: 8, Descending: true}, {Expression: "lower(`Mixed`)"}} + table.Indexes[i.Name] = i + model, err := transform.LowercaseIdentifiers(db) + if err != nil { + t.Fatal(err) + } + parts := model.Schemas[0].Tables[0].Indexes["mixedindex"].KeyParts + if parts[0].Name != "mixed" || parts[1].Expression != "lower(`Mixed`)" || i.KeyParts[0].Name != "Mixed" { + t.Fatalf("lowercase parts: %+v", parts) + } +} + +func TestLowercasePrimaryIndexIsNotDuplicated(t *testing.T) { + db := shopDB() + table := db.Schemas[0].Tables[0] + i := models.InitIndex("PRIMARY", table.Name, table.Schema) + i.Unique = true + i.Type = "btree" + i.Columns = []string{"id"} + i.KeyParts = []models.IndexPart{{Name: "id", Descending: true}} + table.Indexes[i.Name] = i + statements, err := NewWriter(&writers.WriterOptions{StrictConversion: true, LowercaseIdentifiers: true}).Statements(db) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(statements[0], "PRIMARY KEY (`id` DESC)") || strings.Contains(statements[0], "KEY `primary`") { + t.Fatalf("primary key duplicated or attributes lost: %s", statements[0]) + } +} diff --git a/pkg/writers/mysql/diagnostics.go b/pkg/writers/mysql/diagnostics.go new file mode 100644 index 0000000..424d842 --- /dev/null +++ b/pkg/writers/mysql/diagnostics.go @@ -0,0 +1,184 @@ +package mysql + +import ( + "fmt" + "sort" + "strings" + + "git.warky.dev/wdevs/relspecgo/pkg/mariadb" + "git.warky.dev/wdevs/relspecgo/pkg/models" +) + +// Diagnostics cover modeled schema semantics and features discovered by the +// MySQL reader. They do not certify arbitrary SQL expressions or catalog grants. +func conversionDiagnostics(db *models.Database) []string { + var out []string + warn := func(path, feature string) { out = append(out, path+": "+feature) } + unsupported := func(m map[string]any) { + switch values := m["mysql_unsupported"].(type) { + case []string: + for _, v := range values { + warn(v, "unsupported object") + } + case []any: + for _, v := range values { + warn(fmt.Sprint(v), "unsupported object") + } + } + } + metadata := func(path string, m map[string]any, allowed string) { + if len(models.GetDirectives(m)) > 0 { + warn(path, "dialect directives are not emitted") + } + for _, key := range keys(m) { + if strings.HasPrefix(key, "mysql_") && !strings.Contains(" "+allowed+" ", " "+key+" ") { + warn(path, "unsupported metadata "+key+": "+fmt.Sprint(m[key])) + } + } + } + metadata("database", db.Metadata, "mysql_unsupported mysql_sql_mode") + modeWarning := func(path string, m map[string]any) { + if mode := unsupportedMode(metaString(m, "mysql_sql_mode")); mode != "" { + warn(path, "unsupported source SQL mode "+mode+"; generated SQL requires standard quotes and backslash escaping") + } + } + modeWarning("database", db.Metadata) + unsupported(db.Metadata) + if len(db.Domains) > 0 { + warn("database", "domains are not emitted") + } + for _, s := range db.Schemas { + path := s.Name + metadata(path, s.Metadata, "mysql_unsupported mysql_sql_mode") + modeWarning(path, s.Metadata) + unsupported(s.Metadata) + if len(s.Views) > 0 { + warn(path, "views are not emitted") + } + if len(s.Sequences) > 0 { + warn(path, "standalone sequences are not emitted") + } + if len(s.Enums) > 0 { + warn(path, "standalone enum types are not emitted; use native column enum types") + } + if len(s.Scripts) > 0 { + warn(path, "scripts are not emitted") + } + if s.Owner != "" || len(s.Permissions) > 0 { + warn(path, "ownership and permissions are not emitted") + } + for _, t := range s.Tables { + tp := path + "." + t.Name + modeWarning(tp, t.Metadata) + if t.Tablespace != "" { + warn(tp, "tablespace is not emitted") + } + metadata(tp, t.Metadata, "mysql_engine mysql_collation mysql_create_options mysql_auto_increment mysql_sql_mode") + if options := metaString(t.Metadata, "mysql_create_options"); options != "" { + warn(tp, "table CREATE_OPTIONS are not emitted: "+options) + } + for _, name := range keys(t.Columns) { + c := t.Columns[name] + cp := tp + "." + c.Name + metadata(cp, c.Metadata, "mysql_type mysql_type_signature mysql_charset mysql_datetime_precision mysql_default_expression mysql_default_sql mysql_default_value mysql_generation_storage mysql_invisible mysql_on_update") + if _, known := mariadb.CanonicalToMariaDBTypes[strings.ToLower(c.Type)]; !known && metaString(c.Metadata, "mysql_type") == "" { + warn(cp, "unknown type "+c.Type+" maps to "+columnType(c)) + } + if native := metaString(c.Metadata, "mysql_type"); native != "" && metaString(c.Metadata, "mysql_type_signature") != signature(c) { + warn(cp, "edited canonical type/dimensions override native declaration "+native) + } + if c.Type == "timestamptz" { + warn(cp, "time zone semantics are not supported") + } + if c.Type == "jsonb" { + warn(cp, "jsonb storage semantics become JSON") + } + if c.Identity { + warn(cp, "identity semantics are not emitted; use auto_increment") + } + if c.Generated && c.GenerationExpression == "" { + warn(cp, "generated column has no expression") + } + if !c.Generated && c.GenerationExpression != "" { + warn(cp, "generation expression requires generated=true") + } + if c.Generated && (c.Default != nil || c.AutoIncrement) { + warn(cp, "generated column cannot preserve a default or auto_increment") + } + if storage := metaString(c.Metadata, "mysql_generation_storage"); storage != "" && storage != "STORED" && storage != "VIRTUAL" { + warn(cp, "unsupported generation storage "+storage) + } + } + for _, name := range keys(t.Constraints) { + c := t.Constraints[name] + cp := tp + " constraint " + c.Name + metadata(cp, c.Metadata, "mysql_enforced") + if c.Deferrable != nil && *c.Deferrable || c.InitiallyDeferred != nil && *c.InitiallyDeferred { + warn(cp, "deferrable constraints are not supported") + } + switch c.Type { + case models.PrimaryKeyConstraint, models.UniqueConstraint: + case models.CheckConstraint: + if c.Expression == "" { + warn(cp, "check constraint has no expression") + } + case models.ForeignKeyConstraint: + engine := metaString(t.Metadata, "mysql_engine") + if engine != "" && !strings.EqualFold(engine, "InnoDB") && !strings.EqualFold(engine, "NDB") { + warn(cp, "foreign keys are not supported by engine "+engine) + } + for _, action := range []string{c.OnDelete, c.OnUpdate} { + switch strings.ToUpper(action) { + case "", "CASCADE", "SET NULL", "RESTRICT", "NO ACTION": + default: + warn(cp, "unsupported referential action is not emitted: "+action) + } + } + default: + warn(cp, "unsupported constraint type "+string(c.Type)) + } + } + for _, name := range keys(t.Indexes) { + i := t.Indexes[name] + ip := tp + " index " + i.Name + metadata(ip, i.Metadata, "mysql_visible") + if i.Where != "" { + warn(ip, "partial predicate is not emitted") + } + if len(i.Include) > 0 { + warn(ip, "INCLUDE columns are not emitted") + } + if i.Concurrent { + warn(ip, "concurrent creation is not supported") + } + if strings.EqualFold(i.Type, "hash") && !strings.EqualFold(metaString(t.Metadata, "mysql_engine"), "MEMORY") { + warn(ip, "HASH index requires a compatible engine; InnoDB converts it to BTREE") + } + switch strings.ToUpper(i.Type) { + case "", "BTREE", "HASH", "FULLTEXT", "SPATIAL": + default: + warn(ip, "unsupported index type "+i.Type+"; index is not emitted") + } + for _, p := range i.KeyParts { + if (p.Name == "") == (p.Expression == "") { + warn(ip, "key part requires exactly one name or expression") + } + } + } + for _, name := range keys(t.Relationships) { + r := t.Relationships[name] + if t.Constraints[r.ForeignKey] == nil { + warn(tp+" relationship "+r.Name, "no backing foreign key constraint; relationship is not emitted") + } + } + } + } + sort.Strings(out) + unique := out[:0] + for _, d := range out { + if len(unique) == 0 || unique[len(unique)-1] != d { + unique = append(unique, d) + } + } + return unique +} diff --git a/pkg/writers/mysql/live_diagnostics_test.go b/pkg/writers/mysql/live_diagnostics_test.go new file mode 100644 index 0000000..2ecacfe --- /dev/null +++ b/pkg/writers/mysql/live_diagnostics_test.go @@ -0,0 +1,140 @@ +package mysql_test + +import ( + "database/sql" + "fmt" + "git.warky.dev/wdevs/relspecgo/pkg/models" + driver "github.com/go-sql-driver/mysql" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "git.warky.dev/wdevs/relspecgo/pkg/readers" + rmysql "git.warky.dev/wdevs/relspecgo/pkg/readers/mysql" + "git.warky.dev/wdevs/relspecgo/pkg/writers" + wmysql "git.warky.dev/wdevs/relspecgo/pkg/writers/mysql" +) + +func TestLiveUnsupportedObjectsDiagnosed(t *testing.T) { + conn := os.Getenv("RELSPEC_TEST_MYSQL_CONN") + if conn == "" { + t.Skip("RELSPEC_TEST_MYSQL_CONN is not set") + } + db, err := sql.Open("mysql", conn) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { db.Close() }) + prefix := fmt.Sprintf("unsupported_%d", time.Now().UnixNano()) + table, view, trigger := prefix+"_table", prefix+"_view", prefix+"_trigger" + t.Cleanup(func() { + for _, stmt := range []string{"DROP VIEW IF EXISTS " + view, "DROP TRIGGER IF EXISTS " + trigger, "DROP TABLE IF EXISTS " + table} { + if _, err := db.Exec(stmt); err != nil { + t.Error(err) + } + } + }) + for _, stmt := range []string{ + "CREATE TABLE " + table + " (id INT PRIMARY KEY) PARTITION BY HASH(id) PARTITIONS 2", + "CREATE VIEW " + view + " AS SELECT id FROM " + table, + "CREATE TRIGGER " + trigger + " BEFORE INSERT ON " + table + " FOR EACH ROW SET NEW.id = NEW.id + 1", + } { + if _, err := db.Exec(stmt); err != nil { + t.Fatalf("%s: %v", stmt, err) + } + } + model, err := rmysql.NewReader(&readers.ReaderOptions{ConnectionString: conn}).ReadDatabase() + if err != nil { + t.Fatal(err) + } + path := filepath.Join(t.TempDir(), "schema.sql") + if err := os.WriteFile(path, []byte("original"), 0600); err != nil { + t.Fatal(err) + } + err = wmysql.NewWriter(&writers.WriterOptions{StrictConversion: true, OutputPath: path}).WriteDatabase(model) + for _, want := range []string{"partitioned table", view, trigger} { + if err == nil || !strings.Contains(err.Error(), want) { + t.Errorf("missing diagnostic %s: %v", want, err) + } + } + bytes, readErr := os.ReadFile(path) + if readErr != nil || string(bytes) != "original" { + t.Fatalf("strict failure touched file: %s %v", bytes, readErr) + } + // Schema-filtered/library writes retain discovery diagnostics as well. + err = wmysql.NewWriter(&writers.WriterOptions{StrictConversion: true}).WriteSchema(model.Schemas[0]) + if err == nil || !strings.Contains(err.Error(), trigger) { + t.Fatalf("filtered schema lost diagnostics: %v", err) + } + var warnings []string + _, err = wmysql.NewWriter(&writers.WriterOptions{Warning: func(s string) { warnings = append(warnings, s) }}).Statements(model) + if err != nil { + t.Fatal(err) + } + for _, want := range []string{view, trigger, table} { + found := false + for _, warning := range warnings { + if strings.Contains(warning, want) { + found = true + } + } + if !found { + t.Errorf("no warning for %s: %v", want, warnings) + } + } +} + +func TestLiveUnsupportedSQLModeFailsBeforeDDL(t *testing.T) { + conn := os.Getenv("RELSPEC_TEST_MYSQL_CONN") + if conn == "" { + t.Skip("RELSPEC_TEST_MYSQL_CONN is not set") + } + for _, mode := range []string{"NO_BACKSLASH_ESCAPES", "ANSI_QUOTES"} { + t.Run(mode, func(t *testing.T) { + cfg, err := driver.ParseDSN(conn) + if err != nil { + t.Fatal(err) + } + if cfg.Params == nil { + cfg.Params = map[string]string{} + } + cfg.Params["sql_mode"] = "'" + mode + "'" + modeConn := cfg.FormatDSN() + model, err := rmysql.NewReader(&readers.ReaderOptions{ConnectionString: modeConn}).ReadDatabase() + if err != nil { + t.Fatal(err) + } + if _, err := wmysql.NewWriter(&writers.WriterOptions{StrictConversion: true}).Statements(model); err == nil || !strings.Contains(err.Error(), mode) { + t.Fatalf("source mode not diagnosed: %v", err) + } + db, err := sql.Open("mysql", conn) + if err != nil { + t.Fatal(err) + } + defer db.Close() + name := fmt.Sprintf("mode_%d", time.Now().UnixNano()) + defer db.Exec("DROP TABLE IF EXISTS " + name) + target := models.InitDatabase(cfg.DBName) + s := models.InitSchema(cfg.DBName) + target.Schemas = append(target.Schemas, s) + table := models.InitTable(name, s.Name) + s.Tables = append(s.Tables, table) + c := models.InitColumn("id", name, s.Name) + c.Type = "int" + table.Columns[c.Name] = c + err = wmysql.NewWriter(&writers.WriterOptions{StrictConversion: true, Metadata: map[string]any{"connection_string": modeConn}}).WriteDatabase(target) + if err == nil || !strings.Contains(err.Error(), mode) { + t.Fatalf("target mode not rejected: %v", err) + } + var count int + if err := db.QueryRow("SELECT COUNT(*) FROM information_schema.TABLES WHERE TABLE_SCHEMA=? AND TABLE_NAME=?", cfg.DBName, name).Scan(&count); err != nil { + t.Fatal(err) + } + if count != 0 { + t.Fatal("mode rejection occurred after DDL") + } + }) + } +} diff --git a/pkg/writers/mysql/live_roundtrip_test.go b/pkg/writers/mysql/live_roundtrip_test.go new file mode 100644 index 0000000..e0033b6 --- /dev/null +++ b/pkg/writers/mysql/live_roundtrip_test.go @@ -0,0 +1,242 @@ +package mysql_test + +import ( + "database/sql" + "encoding/json" + "fmt" + "gopkg.in/yaml.v3" + "os" + "reflect" + "sort" + "strings" + "testing" + "time" + + "git.warky.dev/wdevs/relspecgo/pkg/models" + "git.warky.dev/wdevs/relspecgo/pkg/readers" + rmysql "git.warky.dev/wdevs/relspecgo/pkg/readers/mysql" + "git.warky.dev/wdevs/relspecgo/pkg/writers" + wmysql "git.warky.dev/wdevs/relspecgo/pkg/writers/mysql" + driver "github.com/go-sql-driver/mysql" +) + +func TestLiveSchemaRoundTrip(t *testing.T) { + conn := os.Getenv("RELSPEC_TEST_MYSQL_CONN") + if conn == "" { + t.Skip("RELSPEC_TEST_MYSQL_CONN is not set") + } + db, err := sql.Open("mysql", conn) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { db.Close() }) + prefix := fmt.Sprintf("rt_%d", time.Now().UnixNano()) + parent, child := prefix+"_zparent", prefix+"_achild" + q := func(n string) string { return "`" + strings.ReplaceAll(n, "`", "``") + "`" } + exec := func(stmt string) { + t.Helper() + if _, err := db.Exec(stmt); err != nil { + t.Fatalf("%v\n%s", err, stmt) + } + } + t.Cleanup(func() { + for _, name := range []string{child + "_copy", child, parent + "_copy", parent} { + if _, err := db.Exec("DROP TABLE IF EXISTS " + q(name)); err != nil { + t.Errorf("cleanup %s: %v", name, err) + } + } + }) + exec("CREATE TABLE " + q(parent) + ` ( + id BIGINT UNSIGNED NOT NULL AUTO_INCREMENT, + code CHAR(3) NOT NULL, + email VARCHAR(100) NOT NULL, + label VARCHAR(64) DEFAULT 'don''t (change)' COMMENT 'column''s comment', + amount DECIMAL(12,4) NOT NULL DEFAULT 3.2500, + medium MEDIUMINT UNSIGNED, + flags BIT(8) DEFAULT b'00001010', + status ENUM('one','two','don''t, split') DEFAULT 'don''t, split', + choices SET('a','b') DEFAULT 'a,b', + stamp TIMESTAMP(6) NULL DEFAULT CURRENT_TIMESTAMP(6) ON UPDATE CURRENT_TIMESTAMP(6), + duration TIME(4), + bytes VARBINARY(16), + binary_def VARBINARY(8) DEFAULT X'00275CFF', + slash VARCHAR(64) DEFAULT 'C:\\temp\\file', + uuid_value CHAR(36) DEFAULT (uuid()), + hidden_col INT INVISIBLE, + dt DATETIME(6) DEFAULT '2020-01-02 03:04:05.123456', + payload MEDIUMBLOB, + doc TEXT DEFAULT ('first; +-- data, not a comment +last'), + generated_amount DECIMAL(13,4) GENERATED ALWAYS AS (amount * 2) STORED, + virtual_label VARCHAR(80) GENERATED ALWAYS AS (concat('prefix:',lower(label))) VIRTUAL, + PRIMARY KEY (id,code), + UNIQUE KEY email_unique (email), + UNIQUE KEY pair_unique (label,email), + UNIQUE KEY unique_expression ((concat(lower(code),'-x')),email), + KEY prefix_desc (label(10) DESC), + KEY expression_key ((lower(email))), + KEY invisible_key (medium) INVISIBLE COMMENT 'index''s comment', + FULLTEXT KEY text_search (doc), + CONSTRAINT ` + q(prefix+"_amount_check") + ` CHECK (amount >= 0), + CONSTRAINT ` + q(prefix+"_optional_check") + ` CHECK (medium > 0 AND status <> 'forbidden;-- value') NOT ENFORCED + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci COMMENT='table''s comment'`) + exec("CREATE TABLE " + q(child) + ` ( + id INT PRIMARY KEY, + parent_id BIGINT UNSIGNED, + parent_code CHAR(3), + parent_email VARCHAR(100), + CONSTRAINT ` + q(prefix+"_composite_fk") + ` FOREIGN KEY (parent_id,parent_code) REFERENCES ` + q(parent) + ` (id,code) ON DELETE SET NULL ON UPDATE CASCADE, + CONSTRAINT ` + q(prefix+"_email_fk") + ` FOREIGN KEY (parent_email) REFERENCES ` + q(parent) + ` (email) ON DELETE CASCADE ON UPDATE RESTRICT + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci`) + readerConfig, err := driver.ParseDSN(conn) + if err != nil { + t.Fatal(err) + } + if readerConfig.Params == nil { + readerConfig.Params = map[string]string{} + } + readerConfig.Params["sql_quote_show_create"] = "0" // reader must ensure quoted native metadata + read := func() *models.Database { + t.Helper() + model, err := rmysql.NewReader(&readers.ReaderOptions{ConnectionString: readerConfig.FormatDSN()}).ReadDatabase() + if err != nil { + t.Fatal(err) + } + return model + } + model := read() + var selected []*models.Table + for _, table := range model.Schemas[0].Tables { + if table.Name == parent || table.Name == child { + selected = append(selected, table) + } + } + if len(selected) != 2 { + t.Fatalf("read %d fixture tables", len(selected)) + } + model.Schemas[0].Tables = selected + model.Metadata = nil + model.Schemas[0].Metadata = nil // unrelated pre-existing test objects are outside this fixture + for _, table := range selected { + if table.Name == parent { + if table.Columns["id"].Type != "uint64" { + t.Fatal("unsigned type lost") + } + if table.Columns["amount"].Precision != 12 || table.Columns["amount"].Scale != 4 { + t.Fatal("decimal dimensions lost") + } + if table.Indexes["expression_key"].KeyParts[0].Expression == "" { + t.Fatal("functional index expression lost") + } + } else if len(table.Relationships) != 2 { + t.Fatalf("foreign key relationships collapsed: %d", len(table.Relationships)) + } + } + // Exercise the serialized model boundary as well as the native reader objects. + encoded, err := json.Marshal(model) + if err != nil { + t.Fatal(err) + } + var target models.Database + if err := json.Unmarshal(encoded, &target); err != nil { + t.Fatal(err) + } + yamlModel, err := yaml.Marshal(&target) + if err != nil { + t.Fatal(err) + } + target = models.Database{} + if err := yaml.Unmarshal(yamlModel, &target); err != nil { + t.Fatal(err) + } + for _, table := range target.Schemas[0].Tables { + table.Name += "_copy" + for _, col := range table.Columns { + col.Table = table.Name + } + for _, index := range table.Indexes { + index.Table = table.Name + } + constraints := map[string]*models.Constraint{} + for _, c := range table.Constraints { + c.Table = table.Name + if c.Type == models.ForeignKeyConstraint { + c.ReferencedTable += "_copy" + } + if c.Type == models.ForeignKeyConstraint || c.Type == models.CheckConstraint { + c.Name += "_copy" + } + constraints[c.Name] = c + } + table.Constraints = constraints + for _, r := range table.Relationships { + r.FromTable += "_copy" + r.ToTable += "_copy" + r.ForeignKey += "_copy" + } + } + opts := &writers.WriterOptions{StrictConversion: true, Metadata: map[string]any{"connection_string": conn}} + if err := wmysql.NewWriter(opts).WriteDatabase(&target); err != nil { + t.Fatal(err) + } + show := func(table string) string { + t.Helper() + var name, ddl string + if err := db.QueryRow("SHOW CREATE TABLE "+q(table)).Scan(&name, &ddl); err != nil { + t.Fatal(err) + } + ddl = strings.ReplaceAll(ddl, "_copy", "") + // Index creation order is not semantic. Keep column and constraint order intact. + lines := strings.Split(ddl, "\n") + var indexes []string + for _, line := range lines { + if strings.HasPrefix(strings.TrimSpace(line), "KEY ") { + indexes = append(indexes, line) + } + } + sort.Strings(indexes) + n := 0 + for i, line := range lines { + if strings.HasPrefix(strings.TrimSpace(line), "KEY ") { + lines[i] = indexes[n] + n++ + } + } + return strings.Join(lines, "\n") + } + for _, name := range []string{parent, child} { + a, b := show(name), show(name+"_copy") + if a != b { + t.Errorf("schema mismatch\nsource:\n%s\ntarget:\n%s", a, b) + } + } + again := read() + for _, table := range again.Schemas[0].Tables { + if table.Name == parent+"_copy" && !reflect.DeepEqual(table.Columns["amount"].Metadata, selected[1].Columns["amount"].Metadata) { + t.Error("type metadata changed on second read") + } + } + // Verify constraints and defaults as behavior, after comparing untouched schemas. + exec("INSERT INTO " + q(parent+"_copy") + " (id,code,email) VALUES (18446744073709551614,'abc','one@example.com')") + exec("INSERT INTO " + q(child+"_copy") + " (id,parent_id,parent_code,parent_email) VALUES (1,18446744073709551614,'abc','one@example.com')") + if _, err := db.Exec("INSERT INTO " + q(child+"_copy") + " (id,parent_id,parent_code) VALUES (2,123,'abc')"); err == nil { + t.Error("foreign key is not enforced") + } + if _, err := db.Exec("UPDATE " + q(parent+"_copy") + " SET amount=-1"); err == nil { + t.Error("check is not enforced") + } + var label, doc, binaryHex, slash string + var flags int + if err := db.QueryRow("SELECT label,doc,flags+0,HEX(binary_def),slash FROM "+q(parent+"_copy")).Scan(&label, &doc, &flags, &binaryHex, &slash); err != nil { + t.Fatal(err) + } + for _, table := range read().Schemas[0].Tables { + if table.Name == parent+"_copy" && table.Metadata["mysql_auto_increment"] != "18446744073709551615" { + t.Errorf("large auto-increment counter lost: %v", table.Metadata["mysql_auto_increment"]) + } + } + if label != "don't (change)" || doc != "first;\n-- data, not a comment\nlast" || flags != 10 || binaryHex != "00275CFF" || slash != `C:\temp\file` { + t.Errorf("defaults corrupted: %q %q %d %s %q", label, doc, flags, binaryHex, slash) + } +} diff --git a/pkg/writers/mysql/writer.go b/pkg/writers/mysql/writer.go index f37e721..ab5f961 100644 --- a/pkg/writers/mysql/writer.go +++ b/pkg/writers/mysql/writer.go @@ -7,13 +7,14 @@ import ( "io" "os" "sort" + "strconv" "strings" - _ "github.com/go-sql-driver/mysql" - "git.warky.dev/wdevs/relspecgo/pkg/mariadb" "git.warky.dev/wdevs/relspecgo/pkg/models" + "git.warky.dev/wdevs/relspecgo/pkg/transform" "git.warky.dev/wdevs/relspecgo/pkg/writers" + _ "github.com/go-sql-driver/mysql" ) type Writer struct { @@ -22,88 +23,192 @@ type Writer struct { } func NewWriter(options *writers.WriterOptions) *Writer { return &Writer{options: options} } -func (w *Writer) WriteDatabase(db *models.Database) error { + +// Statements preflights the whole model, emits conversion diagnostics, and returns +// executable units. No output files or database connections are opened here. +func (w *Writer) Statements(db *models.Database) ([]string, error) { if w.options == nil { - return fmt.Errorf("writer options are required") + return nil, fmt.Errorf("writer options are required") } - if conn, ok := w.options.Metadata["connection_string"].(string); ok && conn != "" { - return w.execute(db, conn) + if db == nil { + return nil, fmt.Errorf("database is required") } - release, err := w.openOutput() - if err != nil { - return err - } - defer release() - return w.writeDatabaseDDL(db) -} - -func (w *Writer) writeDatabaseDDL(db *models.Database) error { - if _, err := fmt.Fprintf(w.writer, "-- MySQL Database Schema\n-- Database: %s\n-- Generated by RelSpec\n\n", db.Name); err != nil { - return err - } - for _, s := range db.Schemas { - if err := w.WriteSchema(s); err != nil { - return err - } - } - return nil -} - -// openOutput points w.writer at the configured destination (output file or -// stdout) when none is set, and returns a func that releases it again. -func (w *Writer) openOutput() (func(), error) { - if w.writer != nil { - return func() {}, nil - } - if w.options != nil && w.options.OutputPath != "" { - f, err := os.Create(w.options.OutputPath) + if w.options.LowercaseIdentifiers { + var err error + db, err = transform.LowercaseIdentifiers(db) if err != nil { return nil, err } - w.writer = f - return func() { - f.Close() - w.writer = nil - }, nil } - w.writer = os.Stdout - return func() { w.writer = nil }, nil + diagnostics := conversionDiagnostics(db) + if len(diagnostics) > 0 && w.options.StrictConversion { + return nil, fmt.Errorf("lossy MySQL conversion rejected:\n- %s", strings.Join(diagnostics, "\n- ")) + } + for _, d := range diagnostics { + if w.options.Warning != nil { + w.options.Warning(d) + } else { + fmt.Fprintln(os.Stderr, "warning: MySQL conversion: "+d) + } + } + return databaseStatements(db), nil } - -func (w *Writer) WriteSchema(s *models.Schema) error { - release, err := w.openOutput() +func (w *Writer) WriteDatabase(db *models.Database) error { + statements, err := w.Statements(db) if err != nil { return err } - defer release() - for _, t := range s.Tables { - if err := w.writeTable(s, t); err != nil { + if conn, ok := w.options.Metadata["connection_string"].(string); ok && conn != "" { + return w.execute(statements, conn) + } + if w.writer != nil { + return writeStatements(w.writer, statements, db.Name) + } + if w.options.OutputPath == "" { + return writeStatements(os.Stdout, statements, db.Name) + } + f, err := os.Create(w.options.OutputPath) + if err != nil { + return err + } + err = writeStatements(f, statements, db.Name) + closeErr := f.Close() + if err != nil { + return err + } + return closeErr +} +func writeStatements(out io.Writer, statements []string, name string) error { + name = strings.NewReplacer("\n", " ", "\r", " ").Replace(name) + if _, err := fmt.Fprintf(out, "-- Database: %s\n", name); err != nil { + return err + } + if _, err := io.WriteString(out, "-- MySQL table schema; generated by RelSpec\n\n"); err != nil { + return err + } + for _, stmt := range statements { + if _, err := io.WriteString(out, stmt); err != nil { return err } } return nil } - +func (w *Writer) WriteSchema(s *models.Schema) error { + return w.WriteDatabase(&models.Database{Schemas: []*models.Schema{s}}) +} func (w *Writer) WriteTable(t *models.Table) error { - release, err := w.openOutput() - if err != nil { - return err + return w.WriteSchema(&models.Schema{Name: "", Tables: []*models.Table{t}}) +} + +func qualified(schema, name string) string { + if schema == "" { + return quote(name) } - defer release() - return w.writeTable(nil, t) + return quote(schema) + "." + quote(name) +} +func keys[T any](m map[string]T) []string { + out := make([]string, 0, len(m)) + for k := range m { + out = append(out, k) + } + sort.Strings(out) + return out +} +func metaString(m map[string]any, key string) string { v, _ := m[key].(string); return v } +func metaBool(m map[string]any, key string, def bool) bool { + v, ok := m[key].(bool) + if ok { + return v + } + return def +} +func signature(c *models.Column) string { + return fmt.Sprintf("%s/%d/%d/%d", c.Type, c.Length, c.Precision, c.Scale) +} +func columnType(c *models.Column) string { + if native := metaString(c.Metadata, "mysql_type"); native != "" && metaString(c.Metadata, "mysql_type_signature") == signature(c) { + return native + } + typ := mariadb.ConvertCanonicalToMariaDB(c.Type) + switch strings.ToLower(c.Type) { + case "string": + if c.Length > 0 { + typ = fmt.Sprintf("VARCHAR(%d)", c.Length) + } + case "decimal": + if c.Precision > 0 { + typ = fmt.Sprintf("DECIMAL(%d,%d)", c.Precision, c.Scale) + } + case "timestamp", "time": + if c.Precision > 0 { + typ += fmt.Sprintf("(%d)", c.Precision) + } + } + return typ } -func (w *Writer) writeTable(s *models.Schema, t *models.Table) error { - _, err := io.WriteString(w.writer, tableStatement(s, t)) - return err +// MySQL string literals use doubled quotes and escaped backslashes. The output +// assumes the server's standard SQL mode (NO_BACKSLASH_ESCAPES is not enabled). +func literal(s string) string { + return "'" + strings.ReplaceAll(strings.ReplaceAll(s, "\\", "\\\\"), "'", "''") + "'" +} +func defaultSQL(c *models.Column) string { + value := fmt.Sprint(c.Default) + if native := metaString(c.Metadata, "mysql_default_sql"); native != "" && metaString(c.Metadata, "mysql_default_value") == value { + return native + } + if strings.HasPrefix(metaString(c.Metadata, "mysql_type"), "bit(") && strings.HasPrefix(value, "b'") && strings.HasSuffix(value, "'") && strings.Trim(value[2:len(value)-1], "01") == "" { + return value + } + if metaBool(c.Metadata, "mysql_default_expression", false) { + if strings.HasPrefix(strings.ToUpper(value), "CURRENT_TIMESTAMP") { + return value + } + return "(" + value + ")" + } + switch strings.ToLower(c.Type) { + case "text", "json", "jsonb", "bytea": + return "(" + literal(value) + ")" + case "string", "uuid", "date", "time", "timestamp", "timestamptz": + return literal(value) + case "bool", "boolean": + if value == "true" { + return "1" + } + if value == "false" { + return "0" + } + } + // Numeric defaults are values, never executable fragments. + if _, err := strconv.ParseFloat(value, 64); err == nil { + return value + } + return literal(value) +} +func indexParts(i *models.Index) string { + if len(i.KeyParts) == 0 { + return quoted(i.Columns) + } + parts := make([]string, 0, len(i.KeyParts)) + for _, p := range i.KeyParts { + s := quote(p.Name) + if p.Expression != "" { + s = "(" + p.Expression + ")" + } + if p.Length > 0 { + s += fmt.Sprintf("(%d)", p.Length) + } + if p.Descending { + s += " DESC" + } + parts = append(parts, s) + } + return strings.Join(parts, ", ") } - func tableStatement(s *models.Schema, t *models.Table) string { - name := t.Name + schema := "" if s != nil { - name = fmt.Sprintf("%s.%s", quote(s.Name), quote(t.Name)) - } else { - name = quote(name) + schema = s.Name } cols := make([]*models.Column, 0, len(t.Columns)) for _, c := range t.Columns { @@ -118,76 +223,196 @@ func tableStatement(s *models.Schema, t *models.Table) string { defs := []string{} pk := []string{} for _, c := range cols { - d := fmt.Sprintf(" %s %s", quote(c.Name), mariadb.ConvertCanonicalToMariaDB(c.Type)) - if c.Length > 0 && strings.EqualFold(c.Type, "string") { - d = fmt.Sprintf(" %s VARCHAR(%d)", quote(c.Name), c.Length) + d := " " + quote(c.Name) + " " + columnType(c) + if charset := metaString(c.Metadata, "mysql_charset"); charset != "" && c.Collation != metaString(t.Metadata, "mysql_collation") { + d += " CHARACTER SET " + quote(charset) + } + if c.Collation != "" && c.Collation != metaString(t.Metadata, "mysql_collation") { + d += " COLLATE " + quote(c.Collation) + } + if c.Generated { + d += " GENERATED ALWAYS AS (" + c.GenerationExpression + ")" + if strings.EqualFold(metaString(c.Metadata, "mysql_generation_storage"), "STORED") { + d += " STORED" + } else { + d += " VIRTUAL" + } } if c.NotNull { d += " NOT NULL" + } else if !c.Generated { + d += " NULL" } if c.AutoIncrement { d += " AUTO_INCREMENT" } - if c.Default != nil { - d += fmt.Sprintf(" DEFAULT %s", writers.QuoteDefaultValue(fmt.Sprint(c.Default), c.Type)) + if c.Default != nil && !c.Generated { + d += " DEFAULT " + defaultSQL(c) + } + if upd := metaString(c.Metadata, "mysql_on_update"); upd != "" { + d += " ON UPDATE " + upd + } + if metaBool(c.Metadata, "mysql_invisible", false) { + d += " INVISIBLE" + } + if c.Comment != "" { + d += " COMMENT " + literal(c.Comment) } defs = append(defs, d) if c.IsPrimaryKey { - pk = append(pk, quote(c.Name)) + pk = append(pk, c.Name) } } - constraintNames := make([]string, 0, len(t.Constraints)) - for name := range t.Constraints { - constraintNames = append(constraintNames, name) - } - sort.Strings(constraintNames) - for _, name := range constraintNames { + for _, name := range keys(t.Constraints) { c := t.Constraints[name] if c.Type == models.PrimaryKeyConstraint { - pk = nil - for _, n := range c.Columns { - pk = append(pk, quote(n)) - } + pk = c.Columns break } } - if len(pk) > 0 { - defs = append(defs, " PRIMARY KEY ("+strings.Join(pk, ", ")+")") - } - for _, name := range constraintNames { - c := t.Constraints[name] - if c.Type == models.UniqueConstraint { - defs = append(defs, fmt.Sprintf(" CONSTRAINT %s UNIQUE (%s)", quote(c.Name), quoted(c.Columns))) + var primaryIndex *models.Index + for name, i := range t.Indexes { + if strings.EqualFold(name, "PRIMARY") { + primaryIndex = i + break } } - return fmt.Sprintf("CREATE TABLE IF NOT EXISTS %s (\n%s\n) ENGINE=InnoDB;\n\n", name, strings.Join(defs, ",\n")) + if i := primaryIndex; i != nil { + defs = append(defs, " PRIMARY KEY ("+indexParts(i)+")") + } else if len(pk) > 0 { + defs = append(defs, " PRIMARY KEY ("+quoted(pk)+")") + } + for _, name := range keys(t.Constraints) { + c := t.Constraints[name] + switch c.Type { + case models.UniqueConstraint: + if t.Indexes[c.Name] == nil { + defs = append(defs, " CONSTRAINT "+quote(c.Name)+" UNIQUE ("+quoted(c.Columns)+")") + } + case models.CheckConstraint: + d := " CONSTRAINT " + quote(c.Name) + " CHECK (" + c.Expression + ")" + if !metaBool(c.Metadata, "mysql_enforced", true) { + d += " NOT ENFORCED" + } + defs = append(defs, d) + } + } + for _, name := range keys(t.Indexes) { + i := t.Indexes[name] + if strings.EqualFold(name, "PRIMARY") { + continue + } + method := strings.ToUpper(i.Type) + if method != "" && method != "BTREE" && method != "HASH" && method != "FULLTEXT" && method != "SPATIAL" { + continue + } + kind := "KEY" + if i.Unique { + kind = "UNIQUE KEY" + } + if method == "FULLTEXT" || method == "SPATIAL" { + kind = method + " KEY" + } + d := " " + kind + " " + quote(i.Name) + " (" + indexParts(i) + ")" + if method == "HASH" { + d += " USING " + method + } + if !metaBool(i.Metadata, "mysql_visible", true) { + d += " INVISIBLE" + } + if i.Comment != "" { + d += " COMMENT " + literal(i.Comment) + } + defs = append(defs, d) + } + engine := metaString(t.Metadata, "mysql_engine") + if engine == "" { + engine = "InnoDB" + } + tail := " ENGINE=" + quote(engine) + if engine == "InnoDB" { + tail = " ENGINE=InnoDB" + } + if coll := metaString(t.Metadata, "mysql_collation"); coll != "" { + tail += " COLLATE=" + quote(coll) + } + if auto := metaString(t.Metadata, "mysql_auto_increment"); auto != "" { + if _, err := strconv.ParseUint(auto, 10, 64); err == nil { + tail += " AUTO_INCREMENT=" + auto + } + } + if t.Comment != "" { + tail += " COMMENT=" + literal(t.Comment) + } + return "CREATE TABLE IF NOT EXISTS " + qualified(schema, t.Name) + " (\n" + strings.Join(defs, ",\n") + "\n)" + tail + ";\n\n" } - func databaseStatements(db *models.Database) []string { var statements []string - for _, schema := range db.Schemas { - for _, table := range schema.Tables { - statements = append(statements, tableStatement(schema, table)) + for _, s := range db.Schemas { + for _, t := range s.Tables { + statements = append(statements, tableStatement(s, t)) + } + } + // References are added after all tables, supporting cycles and forward references. + for _, s := range db.Schemas { + for _, t := range s.Tables { + for _, name := range keys(t.Constraints) { + c := t.Constraints[name] + if c.Type != models.ForeignKeyConstraint { + continue + } + rs := c.ReferencedSchema + if rs == "" { + rs = s.Name + } + stmt := "ALTER TABLE " + qualified(s.Name, t.Name) + " ADD CONSTRAINT " + quote(c.Name) + " FOREIGN KEY (" + quoted(c.Columns) + ") REFERENCES " + qualified(rs, c.ReferencedTable) + " (" + quoted(c.ReferencedColumns) + ")" + if validAction(c.OnDelete) && c.OnDelete != "" { + stmt += " ON DELETE " + strings.ToUpper(c.OnDelete) + } + if validAction(c.OnUpdate) && c.OnUpdate != "" { + stmt += " ON UPDATE " + strings.ToUpper(c.OnUpdate) + } + statements = append(statements, stmt+";\n\n") + } } } return statements } - -func (w *Writer) execute(dbm *models.Database, conn string) error { - statements := databaseStatements(dbm) +func (w *Writer) execute(statements []string, conn string) error { + if w.options.AtomicExecution { + return fmt.Errorf("MySQL DDL implicitly commits; atomic execution is unsupported") + } db, err := sql.Open("mysql", conn) if err != nil { return fmt.Errorf("failed to connect: %w", err) } defer db.Close() - for _, stmt := range statements { - if _, err := db.ExecContext(context.Background(), stmt); err != nil { - return fmt.Errorf("failed to execute SQL: %w", err) + session, err := db.Conn(context.Background()) + if err != nil { + return fmt.Errorf("failed to execute SQL: %w", err) + } + defer session.Close() + var mode string + if err := session.QueryRowContext(context.Background(), "SELECT @@SESSION.sql_mode").Scan(&mode); err != nil { + return fmt.Errorf("read MySQL SQL mode: %w", err) + } + if mode = unsupportedMode(mode); mode != "" { + return fmt.Errorf("MySQL execution requires standard quotes and backslash escaping; %s is enabled", mode) + } + var failures []string + for n, stmt := range statements { + if _, err := session.ExecContext(context.Background(), stmt); err != nil { + failures = append(failures, fmt.Sprintf("statement %d: %v", n+1, err)) + if !w.options.ContinueOnError { + break + } } } + if len(failures) > 0 { + return fmt.Errorf("failed to execute SQL: %s", strings.Join(failures, "; ")) + } return nil } - func quote(s string) string { return "`" + strings.ReplaceAll(s, "`", "``") + "`" } func quoted(xs []string) string { out := make([]string, len(xs)) @@ -196,3 +421,21 @@ func quoted(xs []string) string { } return strings.Join(out, ", ") } + +func validAction(action string) bool { + switch strings.ToUpper(action) { + case "", "CASCADE", "SET NULL", "RESTRICT", "NO ACTION": + return true + } + return false +} + +func unsupportedMode(mode string) string { + var modes []string + for _, unsupported := range []string{"NO_BACKSLASH_ESCAPES", "ANSI_QUOTES"} { + if strings.Contains(strings.ToUpper(mode), unsupported) { + modes = append(modes, unsupported) + } + } + return strings.Join(modes, ", ") +} diff --git a/pkg/writers/writer.go b/pkg/writers/writer.go index ff8caca..ff7b40c 100644 --- a/pkg/writers/writer.go +++ b/pkg/writers/writer.go @@ -56,6 +56,11 @@ const ( // WriterOptions contains common options for writers type WriterOptions struct { + // StrictConversion rejects known lossy MySQL conversions before output or execution. + StrictConversion bool + // Warning receives conversion diagnostics. MySQL writes warnings to stderr when nil. + Warning func(string) + // OutputPath is the path where the output should be written OutputPath string diff --git a/tests/dbtest/README.md b/tests/dbtest/README.md index 5c1458f..70489f4 100644 --- a/tests/dbtest/README.md +++ b/tests/dbtest/README.md @@ -22,7 +22,7 @@ Podman (preferred) or Docker. One tool for postgres, mssql, mysql. |---|---|---|---|---| | postgres | 5439 | `RELSPEC_TEST_PG_CONN` | `tests/postgres/init.sql` | readers/pgsql, writers/pgsql, writers/sqlexec | | mssql | 1439 | `RELSPEC_TEST_MSSQL_CONN` | `test_data/mssql/test_schema.sql` (creates `RelSpecTest`) | readers/mssql, writers/mssql | -| mysql | 3309 | `RELSPEC_TEST_MYSQL_CONN` | `tests/dbtest/init/mysql.sql` | none (no Go driver/reader yet) | +| mysql | 3309 | `RELSPEC_TEST_MYSQL_CONN` | `tests/dbtest/init/mysql.sql` | readers/mysql, writers/mysql | ## Env @@ -43,3 +43,20 @@ Podman (preferred) or Docker. One tool for postgres, mssql, mysql. - Containers are named `relspec-test-`; existing `tests/postgres/*.sh` and `make docker-*` use the same postgres name/port 5439 and remain independent. - mssql needs ~2GB RAM and takes longer to become ready. - Tests skip when the env var is unset. + +## MySQL schema fidelity tests + +Run `./tests/dbtest/dbtest.sh test mysql` for the MySQL 8.4 round-trip and +unsupported-object regressions. They use temporary tables in `relspec_test`, +including checks, functional indexes, composite foreign keys, binary/expression +defaults, and generated columns. The test account needs table, view, and trigger +privileges in that database. The disposable runner enables +`log_bin_trust_function_creators` so binary logging does not require SUPER for +trigger fixtures. For a separately managed test server, disable binary logging +or configure that setting first. No global database-creation privilege is required. +A failed or interrupted test can leave tables named `rt_*` or `unsupported_*`; +restarting the disposable container removes them. + +See [the MySQL capability matrix](../../docs/MYSQL_FIDELITY.md) for the supported +contract and strict conversion options. PostgreSQL/SQL Server tests remain +separate; this suite does not certify MariaDB compatibility. diff --git a/tests/dbtest/dbs/mysql.sh b/tests/dbtest/dbs/mysql.sh index e0ac00e..b8b3840 100644 --- a/tests/dbtest/dbs/mysql.sh +++ b/tests/dbtest/dbs/mysql.sh @@ -8,9 +8,14 @@ DB_ENV=(-e MYSQL_ROOT_PASSWORD=relspec_root_password -e MYSQL_DATABASE=relspec_t DB_INIT_MOUNT="$ROOT/tests/dbtest/init/mysql.sql:/docker-entrypoint-initdb.d/init.sql" DB_CONN_VAR=RELSPEC_TEST_MYSQL_CONN DB_CONN="relspec:relspec_test_password@tcp(localhost:3309)/relspec_test" -DB_DEFAULT_PKGS="" +DB_DEFAULT_PKGS="./pkg/readers/mysql ./pkg/writers/mysql" db_ready() { rt exec "$DB_CONTAINER" mysqladmin ping -h 127.0.0.1 -urelspec -prelspec_test_password --silent >/dev/null 2>&1; } -db_post_init() { :; } +# Configure the final server after readiness; SET GLOBAL during entrypoint init +# only affects its temporary server and is reset before the tests connect. +db_post_init() { + rt exec "$DB_CONTAINER" mysql -uroot -prelspec_root_password \ + -e 'SET GLOBAL log_bin_trust_function_creators = 1' >/dev/null +} db_shell() { rt exec -it "$DB_CONTAINER" mysql -urelspec -prelspec_test_password relspec_test "$@"; } db_exec_file() { rt exec -i "$DB_CONTAINER" mysql -urelspec -prelspec_test_password relspec_test < "$1"; } diff --git a/tests/dbtest/init/mysql.sql b/tests/dbtest/init/mysql.sql index 0eaf714..b20f483 100644 --- a/tests/dbtest/init/mysql.sql +++ b/tests/dbtest/init/mysql.sql @@ -1,5 +1,5 @@ -- Minimal MySQL fixture for relspec container tests. --- Extend when a MySQL reader/writer is added. +-- Reader/writer live tests create additional temporary fixtures. CREATE TABLE users ( id BIGINT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY, email VARCHAR(255) NOT NULL,