fix: preserve MySQL schema fidelity
This commit is contained in:
@@ -79,6 +79,25 @@ block contents. Raw SQL expressions remain executable input. See the
|
||||
[PostgreSQL writer documentation](pkg/writers/pgsql/README.md#identifier-and-value-handling)
|
||||
for library options and details.
|
||||
|
||||
#### MySQL fidelity and strict conversion
|
||||
|
||||
MySQL output now preserves foreign keys and their actions, checks, indexes,
|
||||
unsigned types, dimensions, generated columns, and native defaults. Known
|
||||
unsupported features produce warnings. Add `--strict-conversion` to reject
|
||||
lossy MySQL output before writing a file or executing DDL:
|
||||
|
||||
```bash
|
||||
relspec convert --from mysql --from-conn "$MYSQL_DSN" \
|
||||
--to mysql --to-path schema.sql --strict-conversion
|
||||
```
|
||||
|
||||
Schema jobs can set `options.strict_conversion: true`. This check currently
|
||||
covers MySQL output; it does not guarantee lossless conversion to every format.
|
||||
The live suite covers MySQL 8.4 / InnoDB, including JSON/YAML intermediates.
|
||||
See the [MySQL capability matrix](docs/MYSQL_FIDELITY.md) for unsupported objects,
|
||||
metadata, SQL-mode requirements, and execution limits. Output creates tables in
|
||||
existing databases; it is not a data copy or a rerunnable migration.
|
||||
|
||||
### `batch` — Convert many inputs in one run
|
||||
|
||||
Converts each input independently (one output per input, unlike `--from-list`
|
||||
@@ -202,7 +221,9 @@ references, which are expanded and safety-checked during pre-flight. See
|
||||
|
||||
Use `options.lowercase_identifiers: true` per job, or
|
||||
`relspec job run build-schema --lowercase` to enable lowercasing for schema outputs
|
||||
throughout the selected job plan.
|
||||
throughout the selected job plan. For MySQL schema output,
|
||||
`options.strict_conversion: true` or `--strict-conversion` rejects known lossy
|
||||
conversions before writing.
|
||||
|
||||
### `scripts` — SQL scripts and embedded assets
|
||||
|
||||
|
||||
+20
-4
@@ -181,12 +181,12 @@ Examples:
|
||||
}
|
||||
|
||||
func init() {
|
||||
convertCmd.Flags().StringVar(&convertSourceType, "from", "", "Source format (dbml, dctx, drawdb, graphql, json, yaml, gorm, bun, drizzle, prisma, typeorm, pgsql, sqlite)")
|
||||
convertCmd.Flags().StringVar(&convertSourceType, "from", "", "Source format (dbml, dctx, drawdb, graphql, json, yaml, gorm, bun, drizzle, prisma, typeorm, pgsql, mssql, mysql, sqlite)")
|
||||
convertCmd.Flags().StringVar(&convertSourcePath, "from-path", "", "Source file path (for file-based formats)")
|
||||
convertCmd.Flags().StringVar(&convertSourceConn, "from-conn", "", "Source connection string (for pgsql) or file path (for sqlite)")
|
||||
convertCmd.Flags().StringVar(&convertSourceConn, "from-conn", "", "Source connection string (pgsql, mssql, mysql) or file path (sqlite)")
|
||||
convertCmd.Flags().StringSliceVar(&convertFromList, "from-list", nil, "Comma-separated list of source file paths to read and merge (mutually exclusive with --from-path)")
|
||||
|
||||
convertCmd.Flags().StringVar(&convertTargetType, "to", "", "Target format (dbml, dctx, drawdb, graphql, json, yaml, gorm, bun, drizzle, prisma, typeorm, pgsql)")
|
||||
convertCmd.Flags().StringVar(&convertTargetType, "to", "", "Target format (dbml, dctx, drawdb, graphql, json, yaml, gorm, bun, drizzle, prisma, typeorm, pgsql, mssql, mysql, sqlite)")
|
||||
convertCmd.Flags().StringVar(&convertTargetPath, "to-path", "", "Target output path (file or directory)")
|
||||
convertCmd.Flags().StringVar(&convertPackageName, "package", "", "Package name (for code generation formats like gorm/bun)")
|
||||
convertCmd.Flags().StringVar(&convertSchemaFilter, "schema", "", "Filter to a specific schema by name (required for formats like dctx that only support single schemas)")
|
||||
@@ -449,7 +449,7 @@ func readDatabaseForConvert(dbType, filePath, connString string) (*models.Databa
|
||||
return db, nil
|
||||
}
|
||||
|
||||
func writeDatabase(db *models.Database, dbType, outputPath, packageName, schemaFilter string, flattenSchema bool, nullableTypes, nullableArrays string, continueOnError bool, extraFields string) error {
|
||||
func writeDatabase(db *models.Database, dbType, outputPath, packageName, schemaFilter string, flattenSchema bool, nullableTypes, nullableArrays string, continueOnError bool, extraFields string, strict ...bool) error {
|
||||
if lowercaseIdentifiers {
|
||||
var err error
|
||||
schemaFilter = strings.ToLower(schemaFilter)
|
||||
@@ -462,6 +462,9 @@ func writeDatabase(db *models.Database, dbType, outputPath, packageName, schemaF
|
||||
var writer writers.Writer
|
||||
|
||||
writerOpts := newWriterOptions(outputPath, packageName, flattenSchema, nullableTypes, nullableArrays, continueOnError)
|
||||
if len(strict) > 0 {
|
||||
writerOpts.StrictConversion = writerOpts.StrictConversion || strict[0]
|
||||
}
|
||||
if extraFields != "" {
|
||||
extraFieldsJSON, err := loadExtraFields(dbType, extraFields)
|
||||
if err != nil {
|
||||
@@ -611,6 +614,19 @@ func validateWriteTarget(db *models.Database, dbType, packageName, schemaFilter,
|
||||
"pgsql", "postgres", "postgresql", "sql",
|
||||
"mssql", "sqlserver", "mssql2016", "mssql2017", "mssql2019", "mssql2022",
|
||||
"sqlite", "sqlite3", "prisma", "typeorm", "graphql", "gql":
|
||||
case "mysql", "mariadb":
|
||||
selected := db
|
||||
if schemaFilter != "" {
|
||||
selected = &models.Database{}
|
||||
for _, s := range db.Schemas {
|
||||
if s.Name == schemaFilter {
|
||||
selected.Schemas = append(selected.Schemas, s)
|
||||
}
|
||||
}
|
||||
}
|
||||
if _, err := wmysql.NewWriter(newWriterOptions("", packageName, false, "", "", false)).Statements(selected); err != nil {
|
||||
return err
|
||||
}
|
||||
case "gorm", "bun":
|
||||
if packageName == "" {
|
||||
return fmt.Errorf("package name is required for %s format (use --package flag)", dbType)
|
||||
|
||||
+1
-1
@@ -874,7 +874,7 @@ func writeJobOutput(rj *resolvedJob, db *models.Database, lg *jobLogger) error {
|
||||
lg.logf("writing output: %s (%s)", rj.outputPath, format)
|
||||
|
||||
write := func(target string) error {
|
||||
return writeDatabase(db, format, target, o.Package, o.Schema, o.FlattenSchema, o.Types, o.ArrayNullable, o.ContinueOnError, "")
|
||||
return writeDatabase(db, format, target, o.Package, o.Schema, o.FlattenSchema, o.Types, o.ArrayNullable, o.ContinueOnError, "", o.StrictConversion)
|
||||
}
|
||||
// Single-file formats are written to a temp file and renamed into place so
|
||||
// a failure never leaves a partial or truncated output. Directory-emitting
|
||||
|
||||
@@ -32,5 +32,6 @@ func newWriterOptions(outputPath, packageName string, flattenSchema bool, nullab
|
||||
Prisma7: prisma7,
|
||||
ContinueOnError: continueOnError,
|
||||
StrictDirectives: strictDirectives,
|
||||
StrictConversion: strictConversion,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -18,6 +18,7 @@ var (
|
||||
noVersion bool
|
||||
silent bool
|
||||
strictDirectives bool
|
||||
strictConversion bool
|
||||
lowercaseIdentifiers bool
|
||||
typeMapFlags []string
|
||||
typeMappings map[string]string
|
||||
@@ -62,6 +63,7 @@ func init() {
|
||||
rootCmd.PersistentFlags().StringArrayVar(&typeMapFlags, "type-map", nil, "Override a SQL-to-Go type mapping for bun/gorm output as sqltype=gotype (repeatable), e.g. --type-map uuid=uuid.UUID --type-map numeric=decimal.Decimal")
|
||||
rootCmd.PersistentFlags().BoolVar(&noVersion, "no-version", false, "Suppress the RelSpec version header")
|
||||
rootCmd.PersistentFlags().BoolVar(&silent, "silent", false, "Suppress progress and status messages (errors are still shown)")
|
||||
rootCmd.PersistentFlags().BoolVar(&strictConversion, "strict-conversion", false, "Reject known lossy MySQL schema conversions before writing or executing output")
|
||||
rootCmd.PersistentFlags().BoolVar(&strictDirectives, "strict-directives", false, "Fail on unknown or untranslatable DBML dialect directives (@postgres:, @sqlite:, …)")
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,41 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.warky.dev/wdevs/relspecgo/pkg/jobs"
|
||||
"git.warky.dev/wdevs/relspecgo/pkg/models"
|
||||
)
|
||||
|
||||
func TestStrictConversionOptionAndDryRun(t *testing.T) {
|
||||
prior := strictConversion
|
||||
strictConversion = true
|
||||
t.Cleanup(func() { strictConversion = prior })
|
||||
if flag := rootCmd.PersistentFlags().Lookup("strict-conversion"); flag == nil {
|
||||
t.Fatal("missing --strict-conversion")
|
||||
}
|
||||
db := models.InitDatabase("shop")
|
||||
s := models.InitSchema("shop")
|
||||
table := models.InitTable("users", "shop")
|
||||
col := models.InitColumn("value", "users", "shop")
|
||||
col.Type = "unknown_type"
|
||||
table.Columns[col.Name] = col
|
||||
s.Tables = append(s.Tables, table)
|
||||
db.Schemas = append(db.Schemas, s)
|
||||
if err := validateWriteTarget(db, "mysql", "", "", ""); err == nil || !strings.Contains(err.Error(), "lossy MySQL") {
|
||||
t.Fatalf("dry run bypassed strict validation: %v", err)
|
||||
}
|
||||
strictConversion = false
|
||||
out := filepath.Join(t.TempDir(), "out.sql")
|
||||
rj := &resolvedJob{job: &jobs.Job{Output: &jobs.Output{Format: "mysql"}, Options: jobs.Options{StrictConversion: true}}, outputPath: out}
|
||||
if err := writeJobOutput(rj, db, &jobLogger{file: io.Discard}); err == nil || !strings.Contains(err.Error(), "lossy MySQL") {
|
||||
t.Fatalf("job option bypassed strict validation: %v", err)
|
||||
}
|
||||
if _, err := os.Stat(out); !os.IsNotExist(err) {
|
||||
t.Fatalf("strict job created output: %v", err)
|
||||
}
|
||||
}
|
||||
@@ -191,6 +191,7 @@ jobs:
|
||||
# (file output = full DDL); set options.full_ddl to force full DDL
|
||||
overwrite: false # default false
|
||||
options:
|
||||
strict_conversion: false # reject known lossy MySQL output before writing
|
||||
lowercase_identifiers: false # lowercase names/references; SQL expressions and data keep their case
|
||||
flatten_schema: false
|
||||
schema: public
|
||||
|
||||
@@ -0,0 +1,131 @@
|
||||
# MySQL schema fidelity
|
||||
|
||||
The MySQL reader/writer preserves the table-schema features below. It generates
|
||||
DDL for **new tables in existing databases**, not migrations or data copies.
|
||||
MySQL 8.4 / InnoDB is exercised by the live suite. The reader uses MySQL 8.0.23+
|
||||
metadata (including functional indexes, checks, spatial metadata, and invisible
|
||||
columns); older MySQL versions and the `mariadb` alias are not certified by this
|
||||
suite and may return explicit metadata-query errors.
|
||||
|
||||
## Conversion diagnostics
|
||||
|
||||
MySQL output warns on stderr when it encounters known unsupported features.
|
||||
Add `--strict-conversion` to reject these conversions before opening an output
|
||||
file or connecting to an output database:
|
||||
|
||||
```bash
|
||||
relspec convert --from mysql --from-conn "$MYSQL_DSN" \
|
||||
--to mysql --to-path schema.sql --strict-conversion
|
||||
|
||||
relspec convert --from json --from-path schema.json \
|
||||
--to mysql --to-path schema.sql --strict-conversion --dry-run
|
||||
```
|
||||
|
||||
The flag also applies to MySQL output from `batch`, `split`, and schema jobs.
|
||||
Jobs can set `options.strict_conversion: true`. Library callers use
|
||||
`writers.WriterOptions.StrictConversion`; `Warning func(string)` collects
|
||||
non-strict diagnostics instead of printing them. `mysql.Writer.Statements`
|
||||
performs the same preflight without opening a destination and returns structured
|
||||
SQL statements suitable for execution without splitting on semicolons.
|
||||
|
||||
Strict conversion currently covers **MySQL output**. It is not a general
|
||||
losslessness check for other writers or a validator of arbitrary SQL fragments.
|
||||
Warnings identify the object and omitted or changed feature. Inspect them before
|
||||
using non-strict output; unsupported index types are omitted, while a partial
|
||||
index's predicate and INCLUDE columns cannot be represented in MySQL.
|
||||
|
||||
## MySQL capability matrix
|
||||
|
||||
| Feature | Reader → model | Model → MySQL DDL |
|
||||
| --- | --- | --- |
|
||||
| Ordered columns, nullability, primary/composite keys | Preserved | Preserved |
|
||||
| Unique/composite keys | Preserved, including backing indexes | Preserved once; no duplicate unique index |
|
||||
| Foreign/composite keys and DELETE/UPDATE actions | Preserved | Added after all tables, allowing forward references/cycles |
|
||||
| CHECK and NOT ENFORCED CHECK | Preserved | Preserved |
|
||||
| BTREE, FULLTEXT, SPATIAL indexes | Preserved | Emitted; engine restrictions still apply |
|
||||
| Expression/prefix/descending/invisible index parts | Preserved in `Index.KeyParts` and metadata | Preserved |
|
||||
| HASH indexes | Preserved | Engine-dependent; incompatible engine produces a diagnostic |
|
||||
| Integer unsignedness; mediumint; enum/set; binary/blob/text variants | Canonical type plus native declaration | Native declaration preserved |
|
||||
| Length, numeric precision/scale, fractional seconds | Typed dimensions plus native declaration | Preserved; canonical dimensions also supported |
|
||||
| Defaults, including SQL expressions, bit/binary literals and multiline text | Values plus native SQL spelling from SHOW CREATE | Preserved |
|
||||
| Auto-increment and current next value | Preserved | Preserved |
|
||||
| Stored/virtual generated columns, ON UPDATE, invisible columns | Preserved | Preserved |
|
||||
| Engine, table/column collation, charset and comments | Preserved | Preserved |
|
||||
| Views, triggers, routines, events, partitioned tables | Names recorded in `mysql_unsupported` | Diagnostic; bodies/partition definitions are not emitted |
|
||||
| Nonempty table CREATE_OPTIONS; native parser/storage/engine attributes; spatial SRID | Recorded as unsupported attributes | Diagnostic; not emitted |
|
||||
| Partial/INCLUDE indexes, unsupported index methods | Retained when supplied in the model | Diagnostic |
|
||||
| Deferrable constraints, identity semantics, timestamptz, jsonb storage | Retained when supplied in the model | Diagnostic for unsupported/different semantics |
|
||||
| Standalone sequences/enums/domains, ownership/permissions, scripts and dialect directives | Retained when supplied in the model | Diagnostic; not emitted |
|
||||
| Data, grants, users, server/plugin configuration | Outside the reader's schema contract | Outside the writer's contract |
|
||||
|
||||
Diagnostics depend on the objects visible to the connection account. They cover
|
||||
known modeled/discovered features, not every engine/plugin extension. The server
|
||||
still validates SQL expressions, engine rules, and references at execution time.
|
||||
For example, source and target engines must support the constraints requested.
|
||||
|
||||
## Preservation across intermediate formats
|
||||
|
||||
| Path | Scope |
|
||||
| --- | --- |
|
||||
| MySQL → model → MySQL | Live-tested for the features above on MySQL 8.4 / InnoDB |
|
||||
| MySQL → JSON → model → MySQL | Live-tested; preserves typed index parts and MySQL metadata |
|
||||
| MySQL → YAML → model → MySQL | YAML serializes the same fields/metadata; covered by the live round-trip test |
|
||||
| Other source/target formats | Their own mappings apply; MySQL-specific metadata and semantics are not certified to survive |
|
||||
|
||||
Native declarations and defaults are stored in column metadata as `mysql_type`
|
||||
and `mysql_default_sql`. Reader-generated signatures (`mysql_type_signature`,
|
||||
`mysql_default_value`) prevent these spellings from silently overriding edited
|
||||
types/dimensions/defaults. Editing canonical dimensions invalidates the native
|
||||
type signature, produces a diagnostic, and uses the canonical mapping instead.
|
||||
For a binary default, the logical default is its SQL hex/literal spelling so it
|
||||
survives serialization without invalid UTF-8.
|
||||
|
||||
Other metadata includes `mysql_charset`, `mysql_generation_storage`,
|
||||
`mysql_default_expression`, `mysql_on_update`, `mysql_invisible`, table
|
||||
`mysql_engine` / `mysql_collation` / `mysql_auto_increment`, index
|
||||
`mysql_visible`, and check `mysql_enforced`. Unknown `mysql_*` metadata produces
|
||||
a diagnostic. For a manually authored expression default, set
|
||||
`mysql_default_expression: true`; otherwise strings are literal values.
|
||||
`Index.KeyParts` is authoritative when present; `Index.Columns` remains the
|
||||
compatibility representation for simple indexes.
|
||||
|
||||
Raw native type declarations, default SQL, generation/check/index expressions,
|
||||
and ON UPDATE expressions are executable SQL input. Lowercasing changes names
|
||||
and structured index parts but leaves expressions untouched; update references
|
||||
inside raw SQL yourself when using `--lowercase`.
|
||||
|
||||
## Applying the output
|
||||
|
||||
Create the destination database/schema first. Qualified names are preserved,
|
||||
including foreign-key target schemas. Adjust them explicitly when restoring to
|
||||
a different database. Output assumes standard MySQL backslash escaping;
|
||||
`NO_BACKSLASH_ESCAPES` or `ANSI_QUOTES` on the source produces a conversion
|
||||
diagnostic, and live execution refuses these target modes before DDL. File consumers must use the same
|
||||
supported mode. Other SQL-mode incompatibilities surface as server errors.
|
||||
|
||||
`CREATE TABLE IF NOT EXISTS` does not reconcile existing definitions. Foreign
|
||||
keys use `ALTER TABLE ADD CONSTRAINT` and therefore repeated application can
|
||||
fail on existing constraints. This is creation DDL, not a rerunnable migration.
|
||||
Live execution stops on the first SQL error by default. Explicit
|
||||
`ContinueOnError` attempts later statements and still returns an error if any
|
||||
fail. MySQL DDL implicitly commits; `AtomicExecution` is rejected. PostgreSQL's
|
||||
shared execution-report support is not yet implemented for this writer.
|
||||
|
||||
## Verification
|
||||
|
||||
```bash
|
||||
./tests/dbtest/dbtest.sh test mysql
|
||||
# Or use your own disposable database:
|
||||
RELSPEC_TEST_MYSQL_CONN="$MYSQL_DSN" go test -mod=vendor ./pkg/readers/mysql ./pkg/writers/mysql
|
||||
```
|
||||
|
||||
The live tests compare MySQL's `SHOW CREATE TABLE` definitions, normalizing only
|
||||
temporary target-name suffixes and nonsemantic ordinary-index display order.
|
||||
They re-read the target, exercise constraint enforcement/default values, and
|
||||
verify unsupported objects produce warnings or strict failures without touching
|
||||
an existing output file. Test fixtures are removed after each run.
|
||||
|
||||
Metadata references: [COLUMNS](https://dev.mysql.com/doc/refman/8.4/en/information-schema-columns-table.html),
|
||||
[STATISTICS](https://dev.mysql.com/doc/refman/8.4/en/information-schema-statistics-table.html),
|
||||
[TABLE_CONSTRAINTS](https://dev.mysql.com/doc/refman/8.4/en/information-schema-table-constraints-table.html),
|
||||
and [REFERENTIAL_CONSTRAINTS](https://dev.mysql.com/doc/refman/8.4/en/information-schema-referential-constraints-table.html).
|
||||
@@ -20,6 +20,11 @@ Implementation batches (2026-10-12):
|
||||
- [x] **Requested lowercase option:** `--lowercase` on convert/batch/merge/split/templ/job run, job `options.lowercase_identifiers`, and PostgreSQL `WriterOptions.LowercaseIdentifiers` normalize names and structured references on a copy, rejecting collisions. SQL expressions, defaults, comments, enum values, and data keep their case; callers must adjust references inside raw SQL themselves.
|
||||
- [x] **A08, SQL-aware substitution/execution:** embeds use one-pass literal substitution, preserving dollar characters and preventing replacement rescanning. Named parameters and embed directives respect SQL strings, quoted identifiers, nested comments, casts, and dollar bodies; duplicate/unused embed variables fail. MySQL executes structured generated statements without splitting multiline literals or stripping their contents.
|
||||
|
||||
- [x] **A06, nullable MySQL catalogs:** constraint references, functional-index columns/expressions, index prefix/direction fields, column dimensions/collations/generation metadata, and optional table auto-increment counters use nullable destinations. Queries qualify joined columns. Live primary/unique/composite/functional-key reads pass, including unsigned auto-increment values beyond signed 64-bit range.
|
||||
- [x] **A07, scoped MySQL fidelity and diagnostics:** native types/default spellings and typed dimensions/index parts survive MySQL → model → JSON/YAML → DDL. Output emits foreign keys/actions after all tables, checks/enforcement, unique/ordinary/functional/prefix/descending/invisible/fulltext indexes, generated columns, comments/collations/charsets, ON UPDATE, invisible columns, and auto-increment counters. Known unsupported objects/attributes produce warnings or `--strict-conversion` / job `options.strict_conversion` / library `StrictConversion` failures before output or connection. See [the scoped capability matrix](MYSQL_FIDELITY.md); this covers new-table DDL on MySQL 8.4 / InnoDB, not all engines, MariaDB versions, other format mappings, or migrations.
|
||||
|
||||
A06–A07 verification (2026-10-12): live MySQL 8.4.11 tests compare source/target `SHOW CREATE TABLE` definitions (normalizing temporary target suffixes and ordinary-index display order), re-read the target, and verify FK/check enforcement and binary/bit/multiline/backslash/default values. The suite covers composite/functional unique keys, reverse table dependency order, generated/check/index expressions containing SQL literals, precision, unsigned ranges, and multiple FKs to the same target. It exposed and fixed truncated binary defaults in `information_schema` by extracting native defaults from SQL-aware `SHOW CREATE TABLE` definitions. Native SHOW CREATE also avoids stale auto-increment statistics, and escaped catalog expressions are decoded before rendering. Unsupported views/triggers/partitioned tables produce warnings and strict failures; strict failure preserves existing files and happens before opening an output connection. CLI dry-run and per-job strict options are tested. The MySQL container test runner now selects reader/writer packages by default. `go test -mod=vendor -p 1 ./...` passed with `RELSPEC_TEST_MYSQL_CONN` configured; `go vet -mod=vendor ./...`, CLI strict/dry-run checks, shell syntax, and `git diff --check` passed. The MySQL container runner also passed using its schema-scoped account; its disposable server enables trigger-fixture creation after final-server readiness. PostgreSQL and SQL Server live variables were unset for this batch. Checks used `go1.27.1-X:nodwarf5 linux/amd64`; declared-minimum-toolchain coverage remains A13 work. Disposable MySQL instances were removed after verification. A08 now also has live MySQL coverage for embedded statement delimiters/default contents.
|
||||
|
||||
A05–A08 verification: unit regressions and live PostgreSQL 18 checks cover unusual names, comments/defaults, rerunnable file/statement output, diff migrations, sequences, primary-key replacement, audit functions/triggers, and both standard-string modes. MySQL statement preservation is unit-tested; a live MySQL instance was not used for A08. `go test -mod=vendor -p 1 ./...` passed with `RELSPEC_TEST_PG_CONN` set to the disposable instance; `go vet -mod=vendor ./...`, executable CLI help checks, and `git diff --check` passed. SQL Server tests were not configured for this batch. The disposable instance was removed after verification.
|
||||
|
||||
Additional observation: a parallel live full-suite run hit a PostgreSQL reader catalog race (`could not open relation with OID`) while another package's test dropped schema objects. The writer safely fell back to full DDL and its preflight still stopped all mutations, but the existing test assumed no successful read-only preflights before failure. Running live packages serially passed. Reader resilience to concurrent catalog changes remains outside A05/A08; A10 specifically concerns cancellation/timeouts.
|
||||
@@ -155,6 +160,8 @@ PostgreSQL documents the distinct identifier, string, and dollar-quote rules in
|
||||
|
||||
### A06 — MySQL constraint reading scans nullable metadata into strings
|
||||
|
||||
**Repair status: addressed.** Nullable metadata scans and qualified queries are covered by live MySQL 8.4 round trips with primary, unique, composite FK, and functional-index definitions.
|
||||
|
||||
**P1 / High · Database · Code-confirmed; live MySQL not exercised**
|
||||
|
||||
Evidence: `pkg/readers/mysql/reader.go:181–220`, specifically the `rs`, `rt`, and `rc` string variables passed to `rows.Scan`.
|
||||
@@ -169,6 +176,8 @@ Reference: [MySQL KEY_COLUMN_USAGE documentation](https://dev.mysql.com/doc/refm
|
||||
|
||||
### A07 — MySQL conversion silently loses constraints, indexes, and type detail
|
||||
|
||||
**Repair status: addressed within the [documented MySQL table-schema contract](MYSQL_FIDELITY.md).** Supported features survive live round trips through JSON/YAML; known unsupported modeled/discovered features produce diagnostics or strict conversion failures. MySQL output remains creation DDL for existing databases, with engine/version/SQL-mode limits stated in the capability matrix.
|
||||
|
||||
**P1 / High · Database · Reproduced omissions; additional losses code-confirmed**
|
||||
|
||||
Evidence: `pkg/writers/mysql/writer.go:96–163`; `pkg/readers/mysql/reader.go:153–178`; `pkg/mariadb/datatypes.go`.
|
||||
|
||||
@@ -277,6 +277,7 @@ func parseHumanSize(s string) (int64, error) {
|
||||
|
||||
// Options carries the subset of command flags a job file may set.
|
||||
type Options struct {
|
||||
StrictConversion bool `yaml:"strict_conversion"`
|
||||
LowercaseIdentifiers bool `yaml:"lowercase_identifiers"`
|
||||
FlattenSchema bool `yaml:"flatten_schema"`
|
||||
Schema string `yaml:"schema"`
|
||||
|
||||
+19
-19
@@ -61,6 +61,7 @@ var MariaDBToCanonicalTypes = map[string]string{
|
||||
// CanonicalToMariaDBTypes maps canonical types to MariaDB/MySQL types.
|
||||
var CanonicalToMariaDBTypes = map[string]string{
|
||||
"bool": "TINYINT(1)",
|
||||
"boolean": "TINYINT(1)",
|
||||
"int8": "TINYINT",
|
||||
"int16": "SMALLINT",
|
||||
"int": "INT",
|
||||
@@ -114,20 +115,26 @@ func NormalizeMariaDBType(baseType string) string {
|
||||
// ConvertMariaDBToCanonical converts a MariaDB/MySQL type name to the canonical type.
|
||||
// Strips dimension parameters and normalizes aliases. Defaults to "string".
|
||||
func ConvertMariaDBToCanonical(mariadbType string) string {
|
||||
base := strings.ToLower(strings.TrimSpace(mariadbType))
|
||||
if idx := strings.Index(base, "("); idx >= 0 {
|
||||
base = strings.TrimSpace(base[:idx])
|
||||
lower := strings.ToLower(strings.TrimSpace(mariadbType))
|
||||
base := lower
|
||||
if idx := strings.IndexAny(base, "( "); idx >= 0 {
|
||||
base = base[:idx]
|
||||
}
|
||||
|
||||
if canonical, ok := MariaDBToCanonicalTypes[base]; ok {
|
||||
return canonical
|
||||
}
|
||||
|
||||
// Prefix match for composite types (e.g., "unsigned bigint")
|
||||
for key, canonical := range MariaDBToCanonicalTypes {
|
||||
if strings.HasPrefix(base, key) {
|
||||
return canonical
|
||||
canonical, ok := MariaDBToCanonicalTypes[base]
|
||||
if ok {
|
||||
if strings.Contains(lower, "unsigned") || strings.Contains(lower, "zerofill") {
|
||||
switch canonical {
|
||||
case "int8":
|
||||
return "uint8"
|
||||
case "int16":
|
||||
return "uint16"
|
||||
case "int":
|
||||
return "uint"
|
||||
case "int64":
|
||||
return "uint64"
|
||||
}
|
||||
}
|
||||
return canonical
|
||||
}
|
||||
|
||||
return "string"
|
||||
@@ -145,12 +152,5 @@ func ConvertCanonicalToMariaDB(canonicalType string) string {
|
||||
return mariadbType
|
||||
}
|
||||
|
||||
// Prefix fallback
|
||||
for canonical, mariadb := range CanonicalToMariaDBTypes {
|
||||
if strings.HasPrefix(lower, canonical) {
|
||||
return mariadb
|
||||
}
|
||||
}
|
||||
|
||||
return "VARCHAR(255)"
|
||||
}
|
||||
|
||||
@@ -0,0 +1,13 @@
|
||||
package mariadb
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestIntegerSignedness(t *testing.T) {
|
||||
for _, tc := range []struct{ input, want string }{
|
||||
{"tinyint(3) unsigned", "uint8"}, {"smallint unsigned", "uint16"}, {"mediumint unsigned", "uint"}, {"int unsigned", "uint"}, {"bigint unsigned", "uint64"}, {"bigint zerofill", "uint64"}, {"bigint", "int64"}, {"varchar(32)", "string"},
|
||||
} {
|
||||
if got := ConvertMariaDBToCanonical(tc.input); got != tc.want {
|
||||
t.Errorf("%s: got %s want %s", tc.input, got, tc.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -256,9 +256,19 @@ func (d *Column) SQLName() string {
|
||||
return strings.ToLower(d.Name)
|
||||
}
|
||||
|
||||
// IndexPart is an ordered MySQL index key part. Name and Expression are mutually
|
||||
// exclusive; expressions are executable SQL and are not identifier-normalized.
|
||||
type IndexPart struct {
|
||||
Name string `json:"name,omitempty" yaml:"name,omitempty" xml:"name,omitempty"`
|
||||
Expression string `json:"expression,omitempty" yaml:"expression,omitempty" xml:"expression,omitempty"`
|
||||
Length int `json:"length,omitempty" yaml:"length,omitempty" xml:"length,omitempty"`
|
||||
Descending bool `json:"descending,omitempty" yaml:"descending,omitempty" xml:"descending,omitempty"`
|
||||
}
|
||||
|
||||
// Index represents a database index for optimizing query performance.
|
||||
// Indexes can be unique, partial, or include additional columns.
|
||||
type Index struct {
|
||||
KeyParts []IndexPart `json:"key_parts,omitempty" yaml:"key_parts,omitempty" xml:"key_parts,omitempty"`
|
||||
Name string `json:"name" yaml:"name" xml:"name"`
|
||||
Description string `json:"description,omitempty" yaml:"description,omitempty" xml:"description,omitempty"`
|
||||
Table string `json:"table,omitempty" yaml:"table,omitempty" xml:"table,omitempty"`
|
||||
@@ -318,6 +328,7 @@ func (d *Relationship) SQLName() string {
|
||||
// Constraint represents a database constraint that enforces data integrity rules.
|
||||
// Constraints can be primary keys, foreign keys, unique constraints, check constraints, or not-null constraints.
|
||||
type Constraint struct {
|
||||
Metadata map[string]any `json:"metadata,omitempty" yaml:"metadata,omitempty" xml:"-"`
|
||||
Name string `json:"name" yaml:"name" xml:"name"`
|
||||
Type ConstraintType `json:"type" yaml:"type" xml:"type"`
|
||||
Columns []string `json:"columns" yaml:"columns" xml:"columns"`
|
||||
|
||||
@@ -0,0 +1,194 @@
|
||||
package mysql
|
||||
|
||||
import (
|
||||
"regexp"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// columnDefaults extracts native DEFAULT expressions from SHOW CREATE TABLE.
|
||||
// MySQL can truncate binary defaults in information_schema (for example X'00FF'
|
||||
// appears as just "0x"). SHOW CREATE is authoritative for their SQL spelling.
|
||||
// Separators and whitespace inside quoted strings, identifiers and nested
|
||||
// expressions are never treated as definition/default boundaries.
|
||||
func columnDefaults(create string) map[string]string {
|
||||
out := map[string]string{}
|
||||
depth, start := 0, 0
|
||||
var quoted byte
|
||||
definitions := []string{}
|
||||
for i := 0; i < len(create); i++ {
|
||||
ch := create[i]
|
||||
if quoted != 0 {
|
||||
if ch == '\\' && quoted != '`' && i+1 < len(create) {
|
||||
i++
|
||||
continue
|
||||
}
|
||||
if ch == quoted {
|
||||
if i+1 < len(create) && create[i+1] == quoted {
|
||||
i++
|
||||
} else {
|
||||
quoted = 0
|
||||
}
|
||||
}
|
||||
continue
|
||||
}
|
||||
if ch == '\'' || ch == '"' || ch == '`' {
|
||||
quoted = ch
|
||||
continue
|
||||
}
|
||||
switch ch {
|
||||
case '(':
|
||||
depth++
|
||||
if depth == 1 {
|
||||
start = i + 1
|
||||
}
|
||||
case ',':
|
||||
if depth == 1 {
|
||||
definitions = append(definitions, create[start:i])
|
||||
start = i + 1
|
||||
}
|
||||
case ')':
|
||||
depth--
|
||||
if depth == 0 {
|
||||
definitions = append(definitions, create[start:i])
|
||||
break
|
||||
}
|
||||
}
|
||||
if ch == ')' && depth == 0 {
|
||||
break
|
||||
}
|
||||
}
|
||||
for _, def := range definitions {
|
||||
def = strings.TrimSpace(def)
|
||||
if len(def) == 0 || def[0] != '`' {
|
||||
continue
|
||||
}
|
||||
var name strings.Builder
|
||||
end := 1
|
||||
for end < len(def) {
|
||||
if def[end] == '`' {
|
||||
if end+1 < len(def) && def[end+1] == '`' {
|
||||
name.WriteByte('`')
|
||||
end += 2
|
||||
continue
|
||||
}
|
||||
end++
|
||||
break
|
||||
}
|
||||
name.WriteByte(def[end])
|
||||
end++
|
||||
}
|
||||
// Consume full tokens so DEFAULT inside a generated expression or literal
|
||||
// cannot masquerade as a column default.
|
||||
for end < len(def) {
|
||||
for end < len(def) && isSpace(def[end]) {
|
||||
end++
|
||||
}
|
||||
begin := end
|
||||
end = tokenEnd(def, end)
|
||||
if end == begin {
|
||||
break
|
||||
}
|
||||
if strings.EqualFold(def[begin:end], "DEFAULT") {
|
||||
for end < len(def) && isSpace(def[end]) {
|
||||
end++
|
||||
}
|
||||
finish := tokenEnd(def, end)
|
||||
out[name.String()] = def[end:finish]
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
func isSpace(b byte) bool { return b == ' ' || b == '\n' || b == '\r' || b == '\t' }
|
||||
func tokenEnd(s string, start int) int {
|
||||
depth := 0
|
||||
var quoted byte
|
||||
for i := start; i < len(s); i++ {
|
||||
ch := s[i]
|
||||
if quoted != 0 {
|
||||
if ch == '\\' && quoted != '`' && i+1 < len(s) {
|
||||
i++
|
||||
continue
|
||||
}
|
||||
if ch == quoted {
|
||||
if i+1 < len(s) && s[i+1] == quoted {
|
||||
i++
|
||||
} else {
|
||||
quoted = 0
|
||||
}
|
||||
}
|
||||
continue
|
||||
}
|
||||
if ch == '\'' || ch == '"' || ch == '`' {
|
||||
quoted = ch
|
||||
continue
|
||||
}
|
||||
if ch == '(' {
|
||||
depth++
|
||||
}
|
||||
if ch == ')' {
|
||||
depth--
|
||||
}
|
||||
if depth == 0 && isSpace(ch) {
|
||||
return i
|
||||
}
|
||||
}
|
||||
return len(s)
|
||||
}
|
||||
|
||||
// SHOW CREATE has native options not exposed by STATISTICS/COLUMNS. Ignore
|
||||
// quoted identifiers and values when detecting options outside our contract.
|
||||
func unquotedSQL(create string) string {
|
||||
var code strings.Builder
|
||||
var quoted byte
|
||||
for i := 0; i < len(create); i++ {
|
||||
ch := create[i]
|
||||
if quoted != 0 {
|
||||
if ch == '\\' && quoted != '`' && i+1 < len(create) {
|
||||
i++
|
||||
continue
|
||||
}
|
||||
if ch == quoted {
|
||||
if i+1 < len(create) && create[i+1] == quoted {
|
||||
i++
|
||||
} else {
|
||||
quoted = 0
|
||||
code.WriteByte(' ')
|
||||
}
|
||||
}
|
||||
continue
|
||||
}
|
||||
if ch == '\'' || ch == '"' || ch == '`' {
|
||||
quoted = ch
|
||||
code.WriteByte(' ')
|
||||
continue
|
||||
}
|
||||
code.WriteByte(ch)
|
||||
}
|
||||
return code.String()
|
||||
}
|
||||
|
||||
var autoIncrementOption = regexp.MustCompile(`(?i)\bAUTO_INCREMENT\s*=\s*([0-9]+)`)
|
||||
|
||||
func autoIncrementValue(create string) string {
|
||||
if match := autoIncrementOption.FindStringSubmatch(unquotedSQL(create)); len(match) > 1 {
|
||||
return match[1]
|
||||
}
|
||||
return "1"
|
||||
}
|
||||
func unsupportedNativeOptions(create string) []string {
|
||||
words := strings.FieldsFunc(strings.ToUpper(unquotedSQL(create)), func(r rune) bool { return !(r >= 'A' && r <= 'Z' || r == '_') })
|
||||
var out []string
|
||||
for i, w := range words {
|
||||
switch w {
|
||||
case "ENGINE_ATTRIBUTE", "SECONDARY_ENGINE_ATTRIBUTE", "KEY_BLOCK_SIZE", "COLUMN_FORMAT", "STORAGE":
|
||||
out = append(out, w)
|
||||
case "PARSER":
|
||||
if i > 0 && words[i-1] == "WITH" {
|
||||
out = append(out, "WITH PARSER")
|
||||
}
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,54 @@
|
||||
package mysql
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestUnescapeExpression(t *testing.T) {
|
||||
for _, tc := range []struct{ input, want string }{
|
||||
{`_utf8mb4\'one\\ntwo\'`, `_utf8mb4'one\ntwo'`},
|
||||
{`concat(\'don\\\'t\',\' change\')`, `concat('don\'t',' change')`},
|
||||
{`(uuid())`, `(uuid())`},
|
||||
} {
|
||||
if got := unescapeExpression(tc.input); got != tc.want {
|
||||
t.Errorf("%q: got %q want %q", tc.input, got, tc.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestNativeColumnDefaults(t *testing.T) {
|
||||
create := "CREATE TABLE `odd(table` (\n" +
|
||||
" `a,b` varbinary(8) DEFAULT 0x0027FF,\n" +
|
||||
" `tick``name` varchar(30) DEFAULT 'don''t, (split)',\n" +
|
||||
" `expr` varchar(30) DEFAULT (concat('DEFAULT, ', (uuid()))),\n" +
|
||||
" `generated` varchar(30) GENERATED ALWAYS AS (concat('DEFAULT',`expr`)) VIRTUAL,\n" +
|
||||
" `new\nline` int DEFAULT 3,\n" +
|
||||
" PRIMARY KEY (`a,b`)\n) ENGINE=InnoDB"
|
||||
got := columnDefaults(create)
|
||||
want := map[string]string{"a,b": "0x0027FF", "tick`name": "'don''t, (split)'", "expr": "(concat('DEFAULT, ', (uuid())))", "new\nline": "3"}
|
||||
if len(got) != len(want) {
|
||||
t.Fatalf("defaults: %v", got)
|
||||
}
|
||||
for name, value := range want {
|
||||
if got[name] != value {
|
||||
t.Errorf("%s: got %q want %q", name, got[name], value)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestNativeOptionsIgnoreQuotedValues(t *testing.T) {
|
||||
create := "CREATE TABLE `WITH PARSER` (`STORAGE` int COMMENT 'KEY_BLOCK_SIZE', KEY `ENGINE_ATTRIBUTE` (`STORAGE`) WITH PARSER `ngram`) ENGINE=InnoDB"
|
||||
options := unsupportedNativeOptions(create)
|
||||
if len(options) != 1 || options[0] != "WITH PARSER" {
|
||||
t.Fatalf("native options: %v", options)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNativeAutoIncrementIgnoresQuotedValues(t *testing.T) {
|
||||
for _, tc := range []struct{ create, want string }{
|
||||
{"CREATE TABLE `x` (`id` bigint AUTO_INCREMENT) ENGINE=InnoDB AUTO_INCREMENT=18446744073709551615", "18446744073709551615"},
|
||||
{"CREATE TABLE `x` (`id` bigint AUTO_INCREMENT) COMMENT='AUTO_INCREMENT=12'", "1"},
|
||||
} {
|
||||
if got := autoIncrementValue(tc.create); got != tc.want {
|
||||
t.Errorf("got %s want %s", got, tc.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
+252
-43
@@ -4,9 +4,10 @@ import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
_ "github.com/go-sql-driver/mysql"
|
||||
mysqlDriver "github.com/go-sql-driver/mysql"
|
||||
|
||||
"git.warky.dev/wdevs/relspecgo/pkg/mariadb"
|
||||
"git.warky.dev/wdevs/relspecgo/pkg/models"
|
||||
@@ -31,12 +32,13 @@ func (r *Reader) ReadDatabase() (*models.Database, error) {
|
||||
return nil, fmt.Errorf("failed to connect: %w", err)
|
||||
}
|
||||
defer r.close()
|
||||
var name, version string
|
||||
if err := r.db.QueryRowContext(r.ctx, "SELECT DATABASE()").Scan(&name); err != nil {
|
||||
var name, version, sqlMode string
|
||||
if err := r.db.QueryRowContext(r.ctx, "SELECT DATABASE(), @@SESSION.sql_mode").Scan(&name, &sqlMode); err != nil {
|
||||
return nil, fmt.Errorf("failed to get database name: %w", err)
|
||||
}
|
||||
_ = r.db.QueryRowContext(r.ctx, "SELECT VERSION()").Scan(&version)
|
||||
db := models.InitDatabase(name)
|
||||
db.Metadata["mysql_sql_mode"] = sqlMode
|
||||
db.DatabaseType, db.SourceFormat, db.DatabaseVersion = models.MySQLDatabaseType, "mysql", version
|
||||
schemas, err := r.querySchemas(name)
|
||||
if err != nil {
|
||||
@@ -48,8 +50,10 @@ func (r *Reader) ReadDatabase() (*models.Database, error) {
|
||||
return nil, err
|
||||
}
|
||||
schema.Tables = tables
|
||||
schema.Metadata["mysql_sql_mode"] = sqlMode
|
||||
for _, table := range tables {
|
||||
table.Columns, err = r.queryColumns(schema.Name, table.Name)
|
||||
table.Metadata["mysql_sql_mode"] = sqlMode
|
||||
table.Columns, err = r.queryColumns(schema.Name, table)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -62,6 +66,15 @@ func (r *Reader) ReadDatabase() (*models.Database, error) {
|
||||
return nil, err
|
||||
}
|
||||
table.RefSchema = schema
|
||||
for _, c := range table.Constraints {
|
||||
if c.Type == models.PrimaryKeyConstraint {
|
||||
for _, n := range c.Columns {
|
||||
if col := table.Columns[n]; col != nil {
|
||||
col.IsPrimaryKey = true
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
for _, c := range table.Constraints {
|
||||
if c.Type == models.ForeignKeyConstraint {
|
||||
r.deriveRelationship(table, c)
|
||||
@@ -71,6 +84,9 @@ func (r *Reader) ReadDatabase() (*models.Database, error) {
|
||||
schema.RefDatabase = db
|
||||
db.Schemas = append(db.Schemas, schema)
|
||||
}
|
||||
if err := r.readUnsupported(db, name); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return db, nil
|
||||
}
|
||||
|
||||
@@ -97,7 +113,17 @@ func (r *Reader) ReadTable() (*models.Table, error) {
|
||||
}
|
||||
|
||||
func (r *Reader) connect() error {
|
||||
db, err := sql.Open("mysql", r.options.ConnectionString)
|
||||
cfg, err := mysqlDriver.ParseDSN(r.options.ConnectionString)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if cfg.Params == nil {
|
||||
cfg.Params = map[string]string{}
|
||||
}
|
||||
// SHOW CREATE parsing requires quoted column identifiers, including when the
|
||||
// caller's metadata connection would otherwise disable show-create quoting.
|
||||
cfg.Params["sql_quote_show_create"] = "1"
|
||||
db, err := sql.Open("mysql", cfg.FormatDSN())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -134,61 +160,147 @@ func (r *Reader) querySchemas(current string) ([]*models.Schema, error) {
|
||||
}
|
||||
|
||||
func (r *Reader) queryTables(schema string) ([]*models.Table, error) {
|
||||
rows, err := r.db.QueryContext(r.ctx, "SELECT TABLE_NAME FROM information_schema.TABLES WHERE TABLE_SCHEMA = ? AND TABLE_TYPE = 'BASE TABLE' ORDER BY TABLE_NAME", schema)
|
||||
rows, err := r.db.QueryContext(r.ctx, "SELECT TABLE_NAME, ENGINE, TABLE_COLLATION, TABLE_COMMENT, CREATE_OPTIONS, AUTO_INCREMENT FROM information_schema.TABLES WHERE TABLE_SCHEMA = ? AND TABLE_TYPE = 'BASE TABLE' ORDER BY TABLE_NAME", schema)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
var out []*models.Table
|
||||
for rows.Next() {
|
||||
var n string
|
||||
if err := rows.Scan(&n); err != nil {
|
||||
var n, engine, collation, comment, options string
|
||||
var auto sql.NullString
|
||||
if err := rows.Scan(&n, &engine, &collation, &comment, &options, &auto); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, models.InitTable(n, schema))
|
||||
t := models.InitTable(n, schema)
|
||||
t.Comment = comment
|
||||
t.Metadata["mysql_engine"], t.Metadata["mysql_collation"], t.Metadata["mysql_create_options"] = engine, collation, options
|
||||
if auto.Valid {
|
||||
t.Metadata["mysql_auto_increment"] = auto.String
|
||||
}
|
||||
out = append(out, t)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
func (r *Reader) queryColumns(schema, table string) (map[string]*models.Column, error) {
|
||||
rows, err := r.db.QueryContext(r.ctx, `SELECT COLUMN_NAME, COLUMN_TYPE, IS_NULLABLE, COLUMN_DEFAULT, ORDINAL_POSITION, EXTRA, COLUMN_COMMENT FROM information_schema.COLUMNS WHERE TABLE_SCHEMA = ? AND TABLE_NAME = ? ORDER BY ORDINAL_POSITION`, schema, table)
|
||||
var onUpdateExtra = regexp.MustCompile(`on update current_timestamp(?:\(\d*\))?`)
|
||||
|
||||
func (r *Reader) queryColumns(schema string, table *models.Table) (map[string]*models.Column, error) {
|
||||
rows, err := r.db.QueryContext(r.ctx, `SELECT COLUMN_NAME, COLUMN_TYPE, IS_NULLABLE, COLUMN_DEFAULT, ORDINAL_POSITION, EXTRA, COLUMN_COMMENT,
|
||||
CHARACTER_MAXIMUM_LENGTH, NUMERIC_PRECISION, NUMERIC_SCALE, DATETIME_PRECISION, COLLATION_NAME, CHARACTER_SET_NAME, GENERATION_EXPRESSION, SRS_ID
|
||||
FROM information_schema.COLUMNS WHERE TABLE_SCHEMA=? AND TABLE_NAME=? ORDER BY ORDINAL_POSITION`, schema, table.Name)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return nil, fmt.Errorf("columns %s.%s: %w", schema, table.Name, err)
|
||||
}
|
||||
defer rows.Close()
|
||||
out := map[string]*models.Column{}
|
||||
for rows.Next() {
|
||||
var name, typ, nullable, extra, comment string
|
||||
var def sql.NullString
|
||||
var def, collation, charset, generation sql.NullString
|
||||
var length, precision, scale, datetime, srid sql.NullInt64
|
||||
var pos int
|
||||
if err := rows.Scan(&name, &typ, &nullable, &def, &pos, &extra, &comment); err != nil {
|
||||
if err := rows.Scan(&name, &typ, &nullable, &def, &pos, &extra, &comment, &length, &precision, &scale, &datetime, &collation, &charset, &generation, &srid); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
c := models.InitColumn(name, table, schema)
|
||||
c.Type = r.mapDataType(typ)
|
||||
c.NotNull = strings.EqualFold(nullable, "NO")
|
||||
c.Sequence = uint(pos)
|
||||
c.Comment = comment
|
||||
c := models.InitColumn(name, table.Name, schema)
|
||||
c.Type, c.NotNull, c.Sequence, c.Comment = r.mapDataType(typ), nullable == "NO", uint(pos), comment
|
||||
// The native declaration preserves mediumint, enum/set, binary/blob sizes,
|
||||
// unsigned/zerofill and timestamp versus datetime. Canonical dimensions remain editable.
|
||||
c.Length, c.Precision, c.Scale = int(length.Int64), int(precision.Int64), int(scale.Int64)
|
||||
if datetime.Valid {
|
||||
c.Precision = int(datetime.Int64)
|
||||
}
|
||||
c.Collation = collation.String
|
||||
c.Metadata["mysql_type"] = typ
|
||||
c.Metadata["mysql_type_signature"] = typeSignature(c)
|
||||
c.Metadata["mysql_charset"] = charset.String
|
||||
c.Metadata["mysql_datetime_precision"] = datetime.Int64
|
||||
c.Metadata["mysql_default_expression"] = strings.Contains(extra, "DEFAULT_GENERATED") || strings.HasPrefix(strings.ToLower(def.String), "current_timestamp") && (c.Type == "timestamp")
|
||||
if def.Valid {
|
||||
c.Default = def.String
|
||||
if strings.Contains(extra, "DEFAULT_GENERATED") {
|
||||
c.Default = unescapeExpression(def.String)
|
||||
}
|
||||
if strings.HasPrefix(typ, "bit(") && def.Valid {
|
||||
c.Metadata["mysql_default_expression"] = true
|
||||
}
|
||||
}
|
||||
c.AutoIncrement = strings.Contains(extra, "auto_increment")
|
||||
c.Generated, c.GenerationExpression = generation.String != "", unescapeExpression(generation.String)
|
||||
if strings.Contains(extra, "STORED GENERATED") {
|
||||
c.Metadata["mysql_generation_storage"] = "STORED"
|
||||
} else if c.Generated {
|
||||
c.Metadata["mysql_generation_storage"] = "VIRTUAL"
|
||||
}
|
||||
c.Metadata["mysql_invisible"] = strings.Contains(extra, "INVISIBLE")
|
||||
remaining := onUpdateExtra.ReplaceAllString(strings.ToLower(extra), "")
|
||||
for _, known := range []string{"auto_increment", "default_generated", "stored generated", "virtual generated", "invisible"} {
|
||||
remaining = strings.ReplaceAll(remaining, known, "")
|
||||
}
|
||||
if remaining = strings.TrimSpace(remaining); remaining != "" {
|
||||
c.Metadata["mysql_unsupported_extra"] = remaining
|
||||
}
|
||||
if i := strings.Index(strings.ToLower(extra), "on update "); i >= 0 {
|
||||
c.Metadata["mysql_on_update"] = strings.Fields(extra[i+10:])[0]
|
||||
}
|
||||
if srid.Valid {
|
||||
c.Metadata["mysql_srs_id"] = srid.Int64
|
||||
}
|
||||
c.AutoIncrement = strings.Contains(strings.ToLower(extra), "auto_increment")
|
||||
out[name] = c
|
||||
}
|
||||
return out, rows.Err()
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var tableName, create string
|
||||
qualified := "`" + strings.ReplaceAll(schema, "`", "``") + "`.`" + strings.ReplaceAll(table.Name, "`", "``") + "`"
|
||||
if err := r.db.QueryRowContext(r.ctx, "SHOW CREATE TABLE "+qualified).Scan(&tableName, &create); err != nil {
|
||||
return nil, fmt.Errorf("native defaults %s.%s: %w", schema, table.Name, err)
|
||||
}
|
||||
// information_schema AUTO_INCREMENT may be stale because of statistics caching.
|
||||
for _, c := range out {
|
||||
if c.AutoIncrement {
|
||||
table.Metadata["mysql_auto_increment"] = autoIncrementValue(create)
|
||||
break
|
||||
}
|
||||
}
|
||||
if options := unsupportedNativeOptions(create); len(options) > 0 {
|
||||
table.Metadata["mysql_unsupported_options"] = strings.Join(options, ", ")
|
||||
}
|
||||
for name, sql := range columnDefaults(create) {
|
||||
c := out[name]
|
||||
if c == nil || c.Default == nil {
|
||||
continue
|
||||
}
|
||||
if c.Type == "bytea" {
|
||||
c.Default = sql
|
||||
}
|
||||
c.Metadata["mysql_default_sql"] = sql
|
||||
c.Metadata["mysql_default_value"] = fmt.Sprint(c.Default)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func typeSignature(c *models.Column) string {
|
||||
return fmt.Sprintf("%s/%d/%d/%d", c.Type, c.Length, c.Precision, c.Scale)
|
||||
}
|
||||
|
||||
func (r *Reader) queryConstraints(schema, table string) (map[string]*models.Constraint, error) {
|
||||
rows, err := r.db.QueryContext(r.ctx, `SELECT CONSTRAINT_NAME, CONSTRAINT_TYPE, COLUMN_NAME, REFERENCED_TABLE_SCHEMA, REFERENCED_TABLE_NAME, REFERENCED_COLUMN_NAME, ORDINAL_POSITION FROM information_schema.KEY_COLUMN_USAGE k JOIN information_schema.TABLE_CONSTRAINTS t USING (CONSTRAINT_SCHEMA, TABLE_NAME, CONSTRAINT_NAME) WHERE k.TABLE_SCHEMA=? AND k.TABLE_NAME=? ORDER BY CONSTRAINT_NAME, ORDINAL_POSITION`, schema, table)
|
||||
rows, err := r.db.QueryContext(r.ctx, `SELECT k.CONSTRAINT_NAME, t.CONSTRAINT_TYPE, k.COLUMN_NAME,
|
||||
k.REFERENCED_TABLE_SCHEMA, k.REFERENCED_TABLE_NAME, k.REFERENCED_COLUMN_NAME, k.ORDINAL_POSITION,
|
||||
rc.DELETE_RULE, rc.UPDATE_RULE FROM information_schema.KEY_COLUMN_USAGE k
|
||||
JOIN information_schema.TABLE_CONSTRAINTS t USING (CONSTRAINT_SCHEMA,TABLE_NAME,CONSTRAINT_NAME)
|
||||
LEFT JOIN information_schema.REFERENTIAL_CONSTRAINTS rc ON rc.CONSTRAINT_SCHEMA=k.CONSTRAINT_SCHEMA AND rc.TABLE_NAME=k.TABLE_NAME AND rc.CONSTRAINT_NAME=k.CONSTRAINT_NAME
|
||||
WHERE k.TABLE_SCHEMA=? AND k.TABLE_NAME=? ORDER BY k.CONSTRAINT_NAME,k.ORDINAL_POSITION`, schema, table)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return nil, fmt.Errorf("constraints %s.%s: %w", schema, table, err)
|
||||
}
|
||||
defer rows.Close()
|
||||
out := map[string]*models.Constraint{}
|
||||
for rows.Next() {
|
||||
var name, typ, col, rs, rt, rc string
|
||||
var name, typ string
|
||||
var col, rs, rt, rc, del, upd sql.NullString
|
||||
var pos int
|
||||
if err := rows.Scan(&name, &typ, &col, &rs, &rt, &rc, &pos); err != nil {
|
||||
if err := rows.Scan(&name, &typ, &col, &rs, &rt, &rc, &pos, &del, &upd); err != nil {
|
||||
rows.Close()
|
||||
return nil, err
|
||||
}
|
||||
c := out[name]
|
||||
@@ -201,31 +313,54 @@ func (r *Reader) queryConstraints(schema, table string) (map[string]*models.Cons
|
||||
ct = models.ForeignKeyConstraint
|
||||
}
|
||||
c = models.InitConstraint(name, ct)
|
||||
c.Schema = schema
|
||||
c.Table = table
|
||||
c.ReferencedSchema = rs
|
||||
c.ReferencedTable = rt
|
||||
c.Schema, c.Table = schema, table
|
||||
c.ReferencedSchema, c.ReferencedTable = rs.String, rt.String
|
||||
c.OnDelete, c.OnUpdate = del.String, upd.String
|
||||
out[name] = c
|
||||
}
|
||||
c.Columns = append(c.Columns, col)
|
||||
if rc != "" {
|
||||
c.ReferencedColumns = append(c.ReferencedColumns, rc)
|
||||
if col.Valid {
|
||||
c.Columns = append(c.Columns, col.String)
|
||||
}
|
||||
if rc.Valid {
|
||||
c.ReferencedColumns = append(c.ReferencedColumns, rc.String)
|
||||
}
|
||||
}
|
||||
err = rows.Err()
|
||||
rows.Close()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
rows, err = r.db.QueryContext(r.ctx, `SELECT t.CONSTRAINT_NAME,c.CHECK_CLAUSE,t.ENFORCED FROM information_schema.TABLE_CONSTRAINTS t JOIN information_schema.CHECK_CONSTRAINTS c USING (CONSTRAINT_SCHEMA,CONSTRAINT_NAME) WHERE t.TABLE_SCHEMA=? AND t.TABLE_NAME=? AND t.CONSTRAINT_TYPE='CHECK' ORDER BY t.CONSTRAINT_NAME`, schema, table)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("checks %s.%s (requires MySQL 8.0.16+): %w", schema, table, err)
|
||||
}
|
||||
defer rows.Close()
|
||||
for rows.Next() {
|
||||
var name, expr, enforced string
|
||||
if err := rows.Scan(&name, &expr, &enforced); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
c := models.InitConstraint(name, models.CheckConstraint)
|
||||
c.Schema, c.Table, c.Expression = schema, table, unescapeExpression(expr)
|
||||
c.Metadata = map[string]any{"mysql_enforced": enforced == "YES"}
|
||||
out[name] = c
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
func (r *Reader) queryIndexes(schema, table string) (map[string]*models.Index, error) {
|
||||
rows, err := r.db.QueryContext(r.ctx, `SELECT INDEX_NAME, NON_UNIQUE, COLUMN_NAME, SEQ_IN_INDEX, INDEX_TYPE FROM information_schema.STATISTICS WHERE TABLE_SCHEMA=? AND TABLE_NAME=? AND INDEX_NAME <> 'PRIMARY' ORDER BY INDEX_NAME, SEQ_IN_INDEX`, schema, table)
|
||||
rows, err := r.db.QueryContext(r.ctx, `SELECT INDEX_NAME, NON_UNIQUE, COLUMN_NAME, SEQ_IN_INDEX, INDEX_TYPE, EXPRESSION, SUB_PART, COLLATION, IS_VISIBLE, INDEX_COMMENT FROM information_schema.STATISTICS WHERE TABLE_SCHEMA=? AND TABLE_NAME=? ORDER BY INDEX_NAME,SEQ_IN_INDEX`, schema, table)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return nil, fmt.Errorf("indexes %s.%s (requires MySQL 8.0.13+): %w", schema, table, err)
|
||||
}
|
||||
defer rows.Close()
|
||||
out := map[string]*models.Index{}
|
||||
for rows.Next() {
|
||||
var name, col, typ string
|
||||
var name, typ, visible, comment string
|
||||
var col, expr, direction sql.NullString
|
||||
var prefix sql.NullInt64
|
||||
var non, seq int
|
||||
if err := rows.Scan(&name, &non, &col, &seq, &typ); err != nil {
|
||||
if err := rows.Scan(&name, &non, &col, &seq, &typ, &expr, &prefix, &direction, &visible, &comment); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
i := out[name]
|
||||
@@ -233,22 +368,96 @@ func (r *Reader) queryIndexes(schema, table string) (map[string]*models.Index, e
|
||||
i = models.InitIndex(name, table, schema)
|
||||
i.Unique = non == 0
|
||||
i.Type = strings.ToLower(typ)
|
||||
i.Comment = comment
|
||||
i.Metadata["mysql_visible"] = visible == "YES"
|
||||
out[name] = i
|
||||
}
|
||||
i.Columns = append(i.Columns, col)
|
||||
i.Columns = append(i.Columns, col.String)
|
||||
i.KeyParts = append(i.KeyParts, models.IndexPart{Name: col.String, Expression: unescapeExpression(expr.String), Length: int(prefix.Int64), Descending: direction.String == "D"})
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// Record objects outside the supported table-schema contract rather than silently
|
||||
// discarding them. Writers can warn or reject the conversion before any output.
|
||||
func (r *Reader) readUnsupported(db *models.Database, schema string) error {
|
||||
queries := []struct{ kind, sql string }{
|
||||
{"view", `SELECT TABLE_NAME FROM information_schema.VIEWS WHERE TABLE_SCHEMA=?`},
|
||||
{"trigger", `SELECT TRIGGER_NAME FROM information_schema.TRIGGERS WHERE TRIGGER_SCHEMA=?`},
|
||||
{"routine", `SELECT ROUTINE_NAME FROM information_schema.ROUTINES WHERE ROUTINE_SCHEMA=?`},
|
||||
{"event", `SELECT EVENT_NAME FROM information_schema.EVENTS WHERE EVENT_SCHEMA=?`},
|
||||
{"partitioned table", `SELECT DISTINCT TABLE_NAME FROM information_schema.PARTITIONS WHERE TABLE_SCHEMA=? AND PARTITION_NAME IS NOT NULL`},
|
||||
}
|
||||
var unsupported []string
|
||||
for _, q := range queries {
|
||||
rows, err := r.db.QueryContext(r.ctx, q.sql, schema)
|
||||
if err != nil {
|
||||
return fmt.Errorf("discover %s: %w", q.kind, err)
|
||||
}
|
||||
for rows.Next() {
|
||||
var name string
|
||||
if err := rows.Scan(&name); err != nil {
|
||||
rows.Close()
|
||||
return err
|
||||
}
|
||||
unsupported = append(unsupported, q.kind+" "+schema+"."+name)
|
||||
}
|
||||
err = rows.Err()
|
||||
rows.Close()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if len(unsupported) > 0 {
|
||||
db.Metadata["mysql_unsupported"] = unsupported
|
||||
for _, s := range db.Schemas {
|
||||
s.Metadata["mysql_unsupported"] = unsupported
|
||||
}
|
||||
if r.options.Progress != nil {
|
||||
for _, item := range unsupported {
|
||||
r.options.Progress("warning: unsupported MySQL " + item)
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *Reader) deriveRelationship(t *models.Table, c *models.Constraint) {
|
||||
n := fmt.Sprintf("%s_to_%s", t.Name, c.ReferencedTable)
|
||||
n := fmt.Sprintf("%s_%s", t.Name, c.Name)
|
||||
rel := models.InitRelationship(n, models.OneToMany)
|
||||
rel.FromTable = t.Name
|
||||
rel.FromSchema = t.Schema
|
||||
rel.FromTable, rel.FromSchema = t.Name, t.Schema
|
||||
rel.FromColumns = append([]string(nil), c.Columns...)
|
||||
rel.ToTable = c.ReferencedTable
|
||||
rel.ToSchema = c.ReferencedSchema
|
||||
rel.ToTable, rel.ToSchema = c.ReferencedTable, c.ReferencedSchema
|
||||
rel.ToColumns = append([]string(nil), c.ReferencedColumns...)
|
||||
rel.ForeignKey = c.Name
|
||||
t.Relationships[n] = rel
|
||||
}
|
||||
|
||||
// MySQL information_schema escapes SQL expressions by an additional layer
|
||||
// (including quote delimiters). This covers defaults, generated columns, checks,
|
||||
// and functional indexes. Literal defaults are already values and bypass this.
|
||||
func unescapeExpression(s string) string {
|
||||
var out strings.Builder
|
||||
for i := 0; i < len(s); i++ {
|
||||
if s[i] == '\\' && i+1 < len(s) {
|
||||
i++
|
||||
switch s[i] {
|
||||
case 'n':
|
||||
out.WriteByte('\n')
|
||||
case 'r':
|
||||
out.WriteByte('\r')
|
||||
case 't':
|
||||
out.WriteByte('\t')
|
||||
case '0':
|
||||
out.WriteByte(0)
|
||||
case 'Z':
|
||||
out.WriteByte(26)
|
||||
default:
|
||||
out.WriteByte(s[i])
|
||||
}
|
||||
} else {
|
||||
out.WriteByte(s[i])
|
||||
}
|
||||
}
|
||||
return out.String()
|
||||
}
|
||||
|
||||
@@ -3,12 +3,13 @@ package mysql
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"git.warky.dev/wdevs/relspecgo/pkg/models"
|
||||
"git.warky.dev/wdevs/relspecgo/pkg/readers"
|
||||
)
|
||||
|
||||
func TestReaderMapDataType(t *testing.T) {
|
||||
r := NewReader(&readers.ReaderOptions{})
|
||||
for _, tc := range []struct{ input, want string }{{"varchar(64)", "string"}, {"bigint unsigned", "int64"}, {"datetime", "timestamp"}, {"json", "json"}} {
|
||||
for _, tc := range []struct{ input, want string }{{"varchar(64)", "string"}, {"bigint unsigned", "uint64"}, {"datetime", "timestamp"}, {"json", "json"}} {
|
||||
if got := r.mapDataType(tc.input); got != tc.want {
|
||||
t.Errorf("mapDataType(%q) = %q, want %q", tc.input, got, tc.want)
|
||||
}
|
||||
@@ -20,3 +21,17 @@ func TestReaderRequiresConnectionString(t *testing.T) {
|
||||
t.Fatal("expected missing connection string error")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRelationshipNamesCannotCollapseDistinctForeignKeys(t *testing.T) {
|
||||
table := models.InitTable("t", "app")
|
||||
reader := NewReader(&readers.ReaderOptions{})
|
||||
for _, c := range []*models.Constraint{
|
||||
{Name: "a_to_b", ReferencedTable: "c"},
|
||||
{Name: "a", ReferencedTable: "b_to_c"},
|
||||
} {
|
||||
reader.deriveRelationship(table, c)
|
||||
}
|
||||
if len(table.Relationships) != 2 {
|
||||
t.Fatalf("distinct foreign keys collapsed: %+v", table.Relationships)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,145 @@
|
||||
package mysql
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.warky.dev/wdevs/relspecgo/pkg/models"
|
||||
"git.warky.dev/wdevs/relspecgo/pkg/transform"
|
||||
"git.warky.dev/wdevs/relspecgo/pkg/writers"
|
||||
)
|
||||
|
||||
func TestStrictConversionPreflightsAllObjects(t *testing.T) {
|
||||
db := shopDB()
|
||||
unsupported := models.InitTable("unsupported", "shop")
|
||||
col := models.InitColumn("opaque", "unsupported", "shop")
|
||||
col.Type = "unknown_type"
|
||||
unsupported.Columns[col.Name] = col
|
||||
index := models.InitIndex("partial", "unsupported", "shop")
|
||||
index.Columns = []string{col.Name}
|
||||
index.Where = "opaque IS NOT NULL"
|
||||
index.Include = []string{"other"}
|
||||
unsupported.Indexes[index.Name] = index
|
||||
db.Schemas[0].Tables = append(db.Schemas[0].Tables, unsupported)
|
||||
out := filepath.Join(t.TempDir(), "existing.sql")
|
||||
if err := os.WriteFile(out, []byte("keep existing contents"), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
opts := &writers.WriterOptions{StrictConversion: true, OutputPath: out}
|
||||
err := NewWriter(opts).WriteDatabase(db)
|
||||
for _, want := range []string{"lossy MySQL conversion rejected", "unknown type", "partial predicate", "INCLUDE"} {
|
||||
if err == nil || !strings.Contains(err.Error(), want) {
|
||||
t.Fatalf("expected %s: %v", want, err)
|
||||
}
|
||||
}
|
||||
bytes, err := os.ReadFile(out)
|
||||
if err != nil || string(bytes) != "keep existing contents" {
|
||||
t.Fatalf("destination changed: %q %v", bytes, err)
|
||||
}
|
||||
opts.Metadata = map[string]any{"connection_string": "invalid DSN"}
|
||||
if err := NewWriter(opts).WriteDatabase(db); err == nil || !strings.Contains(err.Error(), "lossy MySQL") {
|
||||
t.Fatalf("connected before strict validation: %v", err)
|
||||
}
|
||||
opts.StrictConversion = false
|
||||
opts.Metadata = nil
|
||||
var warnings []string
|
||||
opts.Warning = func(s string) { warnings = append(warnings, s) }
|
||||
if err := NewWriter(opts).WriteDatabase(db); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(warnings) != 3 {
|
||||
t.Fatalf("warnings: %v", warnings)
|
||||
}
|
||||
}
|
||||
|
||||
func TestUnsupportedModelFeaturesDiagnosed(t *testing.T) {
|
||||
db := shopDB()
|
||||
s := db.Schemas[0]
|
||||
table := s.Tables[0]
|
||||
s.Metadata["mysql_unsupported"] = []any{"trigger shop.audit", "partitioned table shop.archive"}
|
||||
c := table.Columns["id"]
|
||||
c.Type = "timestamptz"
|
||||
c.Identity = true
|
||||
c.Metadata["mysql_srs_id"] = 4326
|
||||
check := models.InitConstraint("deferred", models.CheckConstraint)
|
||||
check.Expression = "id > 0"
|
||||
yes := true
|
||||
check.Deferrable = &yes
|
||||
table.Constraints[check.Name] = check
|
||||
i := models.InitIndex("gin", "users", "shop")
|
||||
i.Type = "gin"
|
||||
i.Columns = []string{"id"}
|
||||
table.Indexes[i.Name] = i
|
||||
_, err := NewWriter(&writers.WriterOptions{StrictConversion: true}).Statements(db)
|
||||
for _, want := range []string{"trigger shop.audit", "partitioned table", "time zone", "identity", "mysql_srs_id", "deferrable", "unsupported index type gin"} {
|
||||
if err == nil || !strings.Contains(err.Error(), want) {
|
||||
t.Errorf("missing diagnostic %q: %v", want, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestCanonicalDimensionsAndNativeEdit(t *testing.T) {
|
||||
db := shopDB()
|
||||
c := db.Schemas[0].Tables[0].Columns["age"]
|
||||
c.Type = "decimal"
|
||||
c.Precision = 12
|
||||
c.Scale = 4
|
||||
if got := columnType(c); got != "DECIMAL(12,4)" {
|
||||
t.Fatal(got)
|
||||
}
|
||||
c.Type = "uint64"
|
||||
c.Precision = 0
|
||||
c.Scale = 0
|
||||
if got := columnType(c); got != "BIGINT UNSIGNED" {
|
||||
t.Fatal(got)
|
||||
}
|
||||
c.Metadata["mysql_type"] = "bigint unsigned"
|
||||
c.Metadata["mysql_type_signature"] = signature(c)
|
||||
c.Type = "int64"
|
||||
_, err := NewWriter(&writers.WriterOptions{StrictConversion: true}).Statements(db)
|
||||
if err == nil || !strings.Contains(err.Error(), "override native declaration") {
|
||||
t.Fatalf("native metadata silently overrode edit: %v", err)
|
||||
}
|
||||
if got := columnType(c); got != "BIGINT" {
|
||||
t.Fatal(got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLowercaseIndexPartsPreserveSQL(t *testing.T) {
|
||||
db := shopDB()
|
||||
table := db.Schemas[0].Tables[0]
|
||||
c := models.InitColumn("Mixed", "users", "shop")
|
||||
c.Type = "string"
|
||||
table.Columns[c.Name] = c
|
||||
i := models.InitIndex("MixedIndex", "users", "shop")
|
||||
i.KeyParts = []models.IndexPart{{Name: "Mixed", Length: 8, Descending: true}, {Expression: "lower(`Mixed`)"}}
|
||||
table.Indexes[i.Name] = i
|
||||
model, err := transform.LowercaseIdentifiers(db)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
parts := model.Schemas[0].Tables[0].Indexes["mixedindex"].KeyParts
|
||||
if parts[0].Name != "mixed" || parts[1].Expression != "lower(`Mixed`)" || i.KeyParts[0].Name != "Mixed" {
|
||||
t.Fatalf("lowercase parts: %+v", parts)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLowercasePrimaryIndexIsNotDuplicated(t *testing.T) {
|
||||
db := shopDB()
|
||||
table := db.Schemas[0].Tables[0]
|
||||
i := models.InitIndex("PRIMARY", table.Name, table.Schema)
|
||||
i.Unique = true
|
||||
i.Type = "btree"
|
||||
i.Columns = []string{"id"}
|
||||
i.KeyParts = []models.IndexPart{{Name: "id", Descending: true}}
|
||||
table.Indexes[i.Name] = i
|
||||
statements, err := NewWriter(&writers.WriterOptions{StrictConversion: true, LowercaseIdentifiers: true}).Statements(db)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !strings.Contains(statements[0], "PRIMARY KEY (`id` DESC)") || strings.Contains(statements[0], "KEY `primary`") {
|
||||
t.Fatalf("primary key duplicated or attributes lost: %s", statements[0])
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,184 @@
|
||||
package mysql
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"git.warky.dev/wdevs/relspecgo/pkg/mariadb"
|
||||
"git.warky.dev/wdevs/relspecgo/pkg/models"
|
||||
)
|
||||
|
||||
// Diagnostics cover modeled schema semantics and features discovered by the
|
||||
// MySQL reader. They do not certify arbitrary SQL expressions or catalog grants.
|
||||
func conversionDiagnostics(db *models.Database) []string {
|
||||
var out []string
|
||||
warn := func(path, feature string) { out = append(out, path+": "+feature) }
|
||||
unsupported := func(m map[string]any) {
|
||||
switch values := m["mysql_unsupported"].(type) {
|
||||
case []string:
|
||||
for _, v := range values {
|
||||
warn(v, "unsupported object")
|
||||
}
|
||||
case []any:
|
||||
for _, v := range values {
|
||||
warn(fmt.Sprint(v), "unsupported object")
|
||||
}
|
||||
}
|
||||
}
|
||||
metadata := func(path string, m map[string]any, allowed string) {
|
||||
if len(models.GetDirectives(m)) > 0 {
|
||||
warn(path, "dialect directives are not emitted")
|
||||
}
|
||||
for _, key := range keys(m) {
|
||||
if strings.HasPrefix(key, "mysql_") && !strings.Contains(" "+allowed+" ", " "+key+" ") {
|
||||
warn(path, "unsupported metadata "+key+": "+fmt.Sprint(m[key]))
|
||||
}
|
||||
}
|
||||
}
|
||||
metadata("database", db.Metadata, "mysql_unsupported mysql_sql_mode")
|
||||
modeWarning := func(path string, m map[string]any) {
|
||||
if mode := unsupportedMode(metaString(m, "mysql_sql_mode")); mode != "" {
|
||||
warn(path, "unsupported source SQL mode "+mode+"; generated SQL requires standard quotes and backslash escaping")
|
||||
}
|
||||
}
|
||||
modeWarning("database", db.Metadata)
|
||||
unsupported(db.Metadata)
|
||||
if len(db.Domains) > 0 {
|
||||
warn("database", "domains are not emitted")
|
||||
}
|
||||
for _, s := range db.Schemas {
|
||||
path := s.Name
|
||||
metadata(path, s.Metadata, "mysql_unsupported mysql_sql_mode")
|
||||
modeWarning(path, s.Metadata)
|
||||
unsupported(s.Metadata)
|
||||
if len(s.Views) > 0 {
|
||||
warn(path, "views are not emitted")
|
||||
}
|
||||
if len(s.Sequences) > 0 {
|
||||
warn(path, "standalone sequences are not emitted")
|
||||
}
|
||||
if len(s.Enums) > 0 {
|
||||
warn(path, "standalone enum types are not emitted; use native column enum types")
|
||||
}
|
||||
if len(s.Scripts) > 0 {
|
||||
warn(path, "scripts are not emitted")
|
||||
}
|
||||
if s.Owner != "" || len(s.Permissions) > 0 {
|
||||
warn(path, "ownership and permissions are not emitted")
|
||||
}
|
||||
for _, t := range s.Tables {
|
||||
tp := path + "." + t.Name
|
||||
modeWarning(tp, t.Metadata)
|
||||
if t.Tablespace != "" {
|
||||
warn(tp, "tablespace is not emitted")
|
||||
}
|
||||
metadata(tp, t.Metadata, "mysql_engine mysql_collation mysql_create_options mysql_auto_increment mysql_sql_mode")
|
||||
if options := metaString(t.Metadata, "mysql_create_options"); options != "" {
|
||||
warn(tp, "table CREATE_OPTIONS are not emitted: "+options)
|
||||
}
|
||||
for _, name := range keys(t.Columns) {
|
||||
c := t.Columns[name]
|
||||
cp := tp + "." + c.Name
|
||||
metadata(cp, c.Metadata, "mysql_type mysql_type_signature mysql_charset mysql_datetime_precision mysql_default_expression mysql_default_sql mysql_default_value mysql_generation_storage mysql_invisible mysql_on_update")
|
||||
if _, known := mariadb.CanonicalToMariaDBTypes[strings.ToLower(c.Type)]; !known && metaString(c.Metadata, "mysql_type") == "" {
|
||||
warn(cp, "unknown type "+c.Type+" maps to "+columnType(c))
|
||||
}
|
||||
if native := metaString(c.Metadata, "mysql_type"); native != "" && metaString(c.Metadata, "mysql_type_signature") != signature(c) {
|
||||
warn(cp, "edited canonical type/dimensions override native declaration "+native)
|
||||
}
|
||||
if c.Type == "timestamptz" {
|
||||
warn(cp, "time zone semantics are not supported")
|
||||
}
|
||||
if c.Type == "jsonb" {
|
||||
warn(cp, "jsonb storage semantics become JSON")
|
||||
}
|
||||
if c.Identity {
|
||||
warn(cp, "identity semantics are not emitted; use auto_increment")
|
||||
}
|
||||
if c.Generated && c.GenerationExpression == "" {
|
||||
warn(cp, "generated column has no expression")
|
||||
}
|
||||
if !c.Generated && c.GenerationExpression != "" {
|
||||
warn(cp, "generation expression requires generated=true")
|
||||
}
|
||||
if c.Generated && (c.Default != nil || c.AutoIncrement) {
|
||||
warn(cp, "generated column cannot preserve a default or auto_increment")
|
||||
}
|
||||
if storage := metaString(c.Metadata, "mysql_generation_storage"); storage != "" && storage != "STORED" && storage != "VIRTUAL" {
|
||||
warn(cp, "unsupported generation storage "+storage)
|
||||
}
|
||||
}
|
||||
for _, name := range keys(t.Constraints) {
|
||||
c := t.Constraints[name]
|
||||
cp := tp + " constraint " + c.Name
|
||||
metadata(cp, c.Metadata, "mysql_enforced")
|
||||
if c.Deferrable != nil && *c.Deferrable || c.InitiallyDeferred != nil && *c.InitiallyDeferred {
|
||||
warn(cp, "deferrable constraints are not supported")
|
||||
}
|
||||
switch c.Type {
|
||||
case models.PrimaryKeyConstraint, models.UniqueConstraint:
|
||||
case models.CheckConstraint:
|
||||
if c.Expression == "" {
|
||||
warn(cp, "check constraint has no expression")
|
||||
}
|
||||
case models.ForeignKeyConstraint:
|
||||
engine := metaString(t.Metadata, "mysql_engine")
|
||||
if engine != "" && !strings.EqualFold(engine, "InnoDB") && !strings.EqualFold(engine, "NDB") {
|
||||
warn(cp, "foreign keys are not supported by engine "+engine)
|
||||
}
|
||||
for _, action := range []string{c.OnDelete, c.OnUpdate} {
|
||||
switch strings.ToUpper(action) {
|
||||
case "", "CASCADE", "SET NULL", "RESTRICT", "NO ACTION":
|
||||
default:
|
||||
warn(cp, "unsupported referential action is not emitted: "+action)
|
||||
}
|
||||
}
|
||||
default:
|
||||
warn(cp, "unsupported constraint type "+string(c.Type))
|
||||
}
|
||||
}
|
||||
for _, name := range keys(t.Indexes) {
|
||||
i := t.Indexes[name]
|
||||
ip := tp + " index " + i.Name
|
||||
metadata(ip, i.Metadata, "mysql_visible")
|
||||
if i.Where != "" {
|
||||
warn(ip, "partial predicate is not emitted")
|
||||
}
|
||||
if len(i.Include) > 0 {
|
||||
warn(ip, "INCLUDE columns are not emitted")
|
||||
}
|
||||
if i.Concurrent {
|
||||
warn(ip, "concurrent creation is not supported")
|
||||
}
|
||||
if strings.EqualFold(i.Type, "hash") && !strings.EqualFold(metaString(t.Metadata, "mysql_engine"), "MEMORY") {
|
||||
warn(ip, "HASH index requires a compatible engine; InnoDB converts it to BTREE")
|
||||
}
|
||||
switch strings.ToUpper(i.Type) {
|
||||
case "", "BTREE", "HASH", "FULLTEXT", "SPATIAL":
|
||||
default:
|
||||
warn(ip, "unsupported index type "+i.Type+"; index is not emitted")
|
||||
}
|
||||
for _, p := range i.KeyParts {
|
||||
if (p.Name == "") == (p.Expression == "") {
|
||||
warn(ip, "key part requires exactly one name or expression")
|
||||
}
|
||||
}
|
||||
}
|
||||
for _, name := range keys(t.Relationships) {
|
||||
r := t.Relationships[name]
|
||||
if t.Constraints[r.ForeignKey] == nil {
|
||||
warn(tp+" relationship "+r.Name, "no backing foreign key constraint; relationship is not emitted")
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
sort.Strings(out)
|
||||
unique := out[:0]
|
||||
for _, d := range out {
|
||||
if len(unique) == 0 || unique[len(unique)-1] != d {
|
||||
unique = append(unique, d)
|
||||
}
|
||||
}
|
||||
return unique
|
||||
}
|
||||
@@ -0,0 +1,140 @@
|
||||
package mysql_test
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"git.warky.dev/wdevs/relspecgo/pkg/models"
|
||||
driver "github.com/go-sql-driver/mysql"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.warky.dev/wdevs/relspecgo/pkg/readers"
|
||||
rmysql "git.warky.dev/wdevs/relspecgo/pkg/readers/mysql"
|
||||
"git.warky.dev/wdevs/relspecgo/pkg/writers"
|
||||
wmysql "git.warky.dev/wdevs/relspecgo/pkg/writers/mysql"
|
||||
)
|
||||
|
||||
func TestLiveUnsupportedObjectsDiagnosed(t *testing.T) {
|
||||
conn := os.Getenv("RELSPEC_TEST_MYSQL_CONN")
|
||||
if conn == "" {
|
||||
t.Skip("RELSPEC_TEST_MYSQL_CONN is not set")
|
||||
}
|
||||
db, err := sql.Open("mysql", conn)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { db.Close() })
|
||||
prefix := fmt.Sprintf("unsupported_%d", time.Now().UnixNano())
|
||||
table, view, trigger := prefix+"_table", prefix+"_view", prefix+"_trigger"
|
||||
t.Cleanup(func() {
|
||||
for _, stmt := range []string{"DROP VIEW IF EXISTS " + view, "DROP TRIGGER IF EXISTS " + trigger, "DROP TABLE IF EXISTS " + table} {
|
||||
if _, err := db.Exec(stmt); err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
}
|
||||
})
|
||||
for _, stmt := range []string{
|
||||
"CREATE TABLE " + table + " (id INT PRIMARY KEY) PARTITION BY HASH(id) PARTITIONS 2",
|
||||
"CREATE VIEW " + view + " AS SELECT id FROM " + table,
|
||||
"CREATE TRIGGER " + trigger + " BEFORE INSERT ON " + table + " FOR EACH ROW SET NEW.id = NEW.id + 1",
|
||||
} {
|
||||
if _, err := db.Exec(stmt); err != nil {
|
||||
t.Fatalf("%s: %v", stmt, err)
|
||||
}
|
||||
}
|
||||
model, err := rmysql.NewReader(&readers.ReaderOptions{ConnectionString: conn}).ReadDatabase()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
path := filepath.Join(t.TempDir(), "schema.sql")
|
||||
if err := os.WriteFile(path, []byte("original"), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
err = wmysql.NewWriter(&writers.WriterOptions{StrictConversion: true, OutputPath: path}).WriteDatabase(model)
|
||||
for _, want := range []string{"partitioned table", view, trigger} {
|
||||
if err == nil || !strings.Contains(err.Error(), want) {
|
||||
t.Errorf("missing diagnostic %s: %v", want, err)
|
||||
}
|
||||
}
|
||||
bytes, readErr := os.ReadFile(path)
|
||||
if readErr != nil || string(bytes) != "original" {
|
||||
t.Fatalf("strict failure touched file: %s %v", bytes, readErr)
|
||||
}
|
||||
// Schema-filtered/library writes retain discovery diagnostics as well.
|
||||
err = wmysql.NewWriter(&writers.WriterOptions{StrictConversion: true}).WriteSchema(model.Schemas[0])
|
||||
if err == nil || !strings.Contains(err.Error(), trigger) {
|
||||
t.Fatalf("filtered schema lost diagnostics: %v", err)
|
||||
}
|
||||
var warnings []string
|
||||
_, err = wmysql.NewWriter(&writers.WriterOptions{Warning: func(s string) { warnings = append(warnings, s) }}).Statements(model)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, want := range []string{view, trigger, table} {
|
||||
found := false
|
||||
for _, warning := range warnings {
|
||||
if strings.Contains(warning, want) {
|
||||
found = true
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
t.Errorf("no warning for %s: %v", want, warnings)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestLiveUnsupportedSQLModeFailsBeforeDDL(t *testing.T) {
|
||||
conn := os.Getenv("RELSPEC_TEST_MYSQL_CONN")
|
||||
if conn == "" {
|
||||
t.Skip("RELSPEC_TEST_MYSQL_CONN is not set")
|
||||
}
|
||||
for _, mode := range []string{"NO_BACKSLASH_ESCAPES", "ANSI_QUOTES"} {
|
||||
t.Run(mode, func(t *testing.T) {
|
||||
cfg, err := driver.ParseDSN(conn)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if cfg.Params == nil {
|
||||
cfg.Params = map[string]string{}
|
||||
}
|
||||
cfg.Params["sql_mode"] = "'" + mode + "'"
|
||||
modeConn := cfg.FormatDSN()
|
||||
model, err := rmysql.NewReader(&readers.ReaderOptions{ConnectionString: modeConn}).ReadDatabase()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := wmysql.NewWriter(&writers.WriterOptions{StrictConversion: true}).Statements(model); err == nil || !strings.Contains(err.Error(), mode) {
|
||||
t.Fatalf("source mode not diagnosed: %v", err)
|
||||
}
|
||||
db, err := sql.Open("mysql", conn)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer db.Close()
|
||||
name := fmt.Sprintf("mode_%d", time.Now().UnixNano())
|
||||
defer db.Exec("DROP TABLE IF EXISTS " + name)
|
||||
target := models.InitDatabase(cfg.DBName)
|
||||
s := models.InitSchema(cfg.DBName)
|
||||
target.Schemas = append(target.Schemas, s)
|
||||
table := models.InitTable(name, s.Name)
|
||||
s.Tables = append(s.Tables, table)
|
||||
c := models.InitColumn("id", name, s.Name)
|
||||
c.Type = "int"
|
||||
table.Columns[c.Name] = c
|
||||
err = wmysql.NewWriter(&writers.WriterOptions{StrictConversion: true, Metadata: map[string]any{"connection_string": modeConn}}).WriteDatabase(target)
|
||||
if err == nil || !strings.Contains(err.Error(), mode) {
|
||||
t.Fatalf("target mode not rejected: %v", err)
|
||||
}
|
||||
var count int
|
||||
if err := db.QueryRow("SELECT COUNT(*) FROM information_schema.TABLES WHERE TABLE_SCHEMA=? AND TABLE_NAME=?", cfg.DBName, name).Scan(&count); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if count != 0 {
|
||||
t.Fatal("mode rejection occurred after DDL")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,242 @@
|
||||
package mysql_test
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"gopkg.in/yaml.v3"
|
||||
"os"
|
||||
"reflect"
|
||||
"sort"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.warky.dev/wdevs/relspecgo/pkg/models"
|
||||
"git.warky.dev/wdevs/relspecgo/pkg/readers"
|
||||
rmysql "git.warky.dev/wdevs/relspecgo/pkg/readers/mysql"
|
||||
"git.warky.dev/wdevs/relspecgo/pkg/writers"
|
||||
wmysql "git.warky.dev/wdevs/relspecgo/pkg/writers/mysql"
|
||||
driver "github.com/go-sql-driver/mysql"
|
||||
)
|
||||
|
||||
func TestLiveSchemaRoundTrip(t *testing.T) {
|
||||
conn := os.Getenv("RELSPEC_TEST_MYSQL_CONN")
|
||||
if conn == "" {
|
||||
t.Skip("RELSPEC_TEST_MYSQL_CONN is not set")
|
||||
}
|
||||
db, err := sql.Open("mysql", conn)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { db.Close() })
|
||||
prefix := fmt.Sprintf("rt_%d", time.Now().UnixNano())
|
||||
parent, child := prefix+"_zparent", prefix+"_achild"
|
||||
q := func(n string) string { return "`" + strings.ReplaceAll(n, "`", "``") + "`" }
|
||||
exec := func(stmt string) {
|
||||
t.Helper()
|
||||
if _, err := db.Exec(stmt); err != nil {
|
||||
t.Fatalf("%v\n%s", err, stmt)
|
||||
}
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
for _, name := range []string{child + "_copy", child, parent + "_copy", parent} {
|
||||
if _, err := db.Exec("DROP TABLE IF EXISTS " + q(name)); err != nil {
|
||||
t.Errorf("cleanup %s: %v", name, err)
|
||||
}
|
||||
}
|
||||
})
|
||||
exec("CREATE TABLE " + q(parent) + ` (
|
||||
id BIGINT UNSIGNED NOT NULL AUTO_INCREMENT,
|
||||
code CHAR(3) NOT NULL,
|
||||
email VARCHAR(100) NOT NULL,
|
||||
label VARCHAR(64) DEFAULT 'don''t (change)' COMMENT 'column''s comment',
|
||||
amount DECIMAL(12,4) NOT NULL DEFAULT 3.2500,
|
||||
medium MEDIUMINT UNSIGNED,
|
||||
flags BIT(8) DEFAULT b'00001010',
|
||||
status ENUM('one','two','don''t, split') DEFAULT 'don''t, split',
|
||||
choices SET('a','b') DEFAULT 'a,b',
|
||||
stamp TIMESTAMP(6) NULL DEFAULT CURRENT_TIMESTAMP(6) ON UPDATE CURRENT_TIMESTAMP(6),
|
||||
duration TIME(4),
|
||||
bytes VARBINARY(16),
|
||||
binary_def VARBINARY(8) DEFAULT X'00275CFF',
|
||||
slash VARCHAR(64) DEFAULT 'C:\\temp\\file',
|
||||
uuid_value CHAR(36) DEFAULT (uuid()),
|
||||
hidden_col INT INVISIBLE,
|
||||
dt DATETIME(6) DEFAULT '2020-01-02 03:04:05.123456',
|
||||
payload MEDIUMBLOB,
|
||||
doc TEXT DEFAULT ('first;
|
||||
-- data, not a comment
|
||||
last'),
|
||||
generated_amount DECIMAL(13,4) GENERATED ALWAYS AS (amount * 2) STORED,
|
||||
virtual_label VARCHAR(80) GENERATED ALWAYS AS (concat('prefix:',lower(label))) VIRTUAL,
|
||||
PRIMARY KEY (id,code),
|
||||
UNIQUE KEY email_unique (email),
|
||||
UNIQUE KEY pair_unique (label,email),
|
||||
UNIQUE KEY unique_expression ((concat(lower(code),'-x')),email),
|
||||
KEY prefix_desc (label(10) DESC),
|
||||
KEY expression_key ((lower(email))),
|
||||
KEY invisible_key (medium) INVISIBLE COMMENT 'index''s comment',
|
||||
FULLTEXT KEY text_search (doc),
|
||||
CONSTRAINT ` + q(prefix+"_amount_check") + ` CHECK (amount >= 0),
|
||||
CONSTRAINT ` + q(prefix+"_optional_check") + ` CHECK (medium > 0 AND status <> 'forbidden;-- value') NOT ENFORCED
|
||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci COMMENT='table''s comment'`)
|
||||
exec("CREATE TABLE " + q(child) + ` (
|
||||
id INT PRIMARY KEY,
|
||||
parent_id BIGINT UNSIGNED,
|
||||
parent_code CHAR(3),
|
||||
parent_email VARCHAR(100),
|
||||
CONSTRAINT ` + q(prefix+"_composite_fk") + ` FOREIGN KEY (parent_id,parent_code) REFERENCES ` + q(parent) + ` (id,code) ON DELETE SET NULL ON UPDATE CASCADE,
|
||||
CONSTRAINT ` + q(prefix+"_email_fk") + ` FOREIGN KEY (parent_email) REFERENCES ` + q(parent) + ` (email) ON DELETE CASCADE ON UPDATE RESTRICT
|
||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci`)
|
||||
readerConfig, err := driver.ParseDSN(conn)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if readerConfig.Params == nil {
|
||||
readerConfig.Params = map[string]string{}
|
||||
}
|
||||
readerConfig.Params["sql_quote_show_create"] = "0" // reader must ensure quoted native metadata
|
||||
read := func() *models.Database {
|
||||
t.Helper()
|
||||
model, err := rmysql.NewReader(&readers.ReaderOptions{ConnectionString: readerConfig.FormatDSN()}).ReadDatabase()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return model
|
||||
}
|
||||
model := read()
|
||||
var selected []*models.Table
|
||||
for _, table := range model.Schemas[0].Tables {
|
||||
if table.Name == parent || table.Name == child {
|
||||
selected = append(selected, table)
|
||||
}
|
||||
}
|
||||
if len(selected) != 2 {
|
||||
t.Fatalf("read %d fixture tables", len(selected))
|
||||
}
|
||||
model.Schemas[0].Tables = selected
|
||||
model.Metadata = nil
|
||||
model.Schemas[0].Metadata = nil // unrelated pre-existing test objects are outside this fixture
|
||||
for _, table := range selected {
|
||||
if table.Name == parent {
|
||||
if table.Columns["id"].Type != "uint64" {
|
||||
t.Fatal("unsigned type lost")
|
||||
}
|
||||
if table.Columns["amount"].Precision != 12 || table.Columns["amount"].Scale != 4 {
|
||||
t.Fatal("decimal dimensions lost")
|
||||
}
|
||||
if table.Indexes["expression_key"].KeyParts[0].Expression == "" {
|
||||
t.Fatal("functional index expression lost")
|
||||
}
|
||||
} else if len(table.Relationships) != 2 {
|
||||
t.Fatalf("foreign key relationships collapsed: %d", len(table.Relationships))
|
||||
}
|
||||
}
|
||||
// Exercise the serialized model boundary as well as the native reader objects.
|
||||
encoded, err := json.Marshal(model)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var target models.Database
|
||||
if err := json.Unmarshal(encoded, &target); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
yamlModel, err := yaml.Marshal(&target)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
target = models.Database{}
|
||||
if err := yaml.Unmarshal(yamlModel, &target); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, table := range target.Schemas[0].Tables {
|
||||
table.Name += "_copy"
|
||||
for _, col := range table.Columns {
|
||||
col.Table = table.Name
|
||||
}
|
||||
for _, index := range table.Indexes {
|
||||
index.Table = table.Name
|
||||
}
|
||||
constraints := map[string]*models.Constraint{}
|
||||
for _, c := range table.Constraints {
|
||||
c.Table = table.Name
|
||||
if c.Type == models.ForeignKeyConstraint {
|
||||
c.ReferencedTable += "_copy"
|
||||
}
|
||||
if c.Type == models.ForeignKeyConstraint || c.Type == models.CheckConstraint {
|
||||
c.Name += "_copy"
|
||||
}
|
||||
constraints[c.Name] = c
|
||||
}
|
||||
table.Constraints = constraints
|
||||
for _, r := range table.Relationships {
|
||||
r.FromTable += "_copy"
|
||||
r.ToTable += "_copy"
|
||||
r.ForeignKey += "_copy"
|
||||
}
|
||||
}
|
||||
opts := &writers.WriterOptions{StrictConversion: true, Metadata: map[string]any{"connection_string": conn}}
|
||||
if err := wmysql.NewWriter(opts).WriteDatabase(&target); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
show := func(table string) string {
|
||||
t.Helper()
|
||||
var name, ddl string
|
||||
if err := db.QueryRow("SHOW CREATE TABLE "+q(table)).Scan(&name, &ddl); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ddl = strings.ReplaceAll(ddl, "_copy", "")
|
||||
// Index creation order is not semantic. Keep column and constraint order intact.
|
||||
lines := strings.Split(ddl, "\n")
|
||||
var indexes []string
|
||||
for _, line := range lines {
|
||||
if strings.HasPrefix(strings.TrimSpace(line), "KEY ") {
|
||||
indexes = append(indexes, line)
|
||||
}
|
||||
}
|
||||
sort.Strings(indexes)
|
||||
n := 0
|
||||
for i, line := range lines {
|
||||
if strings.HasPrefix(strings.TrimSpace(line), "KEY ") {
|
||||
lines[i] = indexes[n]
|
||||
n++
|
||||
}
|
||||
}
|
||||
return strings.Join(lines, "\n")
|
||||
}
|
||||
for _, name := range []string{parent, child} {
|
||||
a, b := show(name), show(name+"_copy")
|
||||
if a != b {
|
||||
t.Errorf("schema mismatch\nsource:\n%s\ntarget:\n%s", a, b)
|
||||
}
|
||||
}
|
||||
again := read()
|
||||
for _, table := range again.Schemas[0].Tables {
|
||||
if table.Name == parent+"_copy" && !reflect.DeepEqual(table.Columns["amount"].Metadata, selected[1].Columns["amount"].Metadata) {
|
||||
t.Error("type metadata changed on second read")
|
||||
}
|
||||
}
|
||||
// Verify constraints and defaults as behavior, after comparing untouched schemas.
|
||||
exec("INSERT INTO " + q(parent+"_copy") + " (id,code,email) VALUES (18446744073709551614,'abc','one@example.com')")
|
||||
exec("INSERT INTO " + q(child+"_copy") + " (id,parent_id,parent_code,parent_email) VALUES (1,18446744073709551614,'abc','one@example.com')")
|
||||
if _, err := db.Exec("INSERT INTO " + q(child+"_copy") + " (id,parent_id,parent_code) VALUES (2,123,'abc')"); err == nil {
|
||||
t.Error("foreign key is not enforced")
|
||||
}
|
||||
if _, err := db.Exec("UPDATE " + q(parent+"_copy") + " SET amount=-1"); err == nil {
|
||||
t.Error("check is not enforced")
|
||||
}
|
||||
var label, doc, binaryHex, slash string
|
||||
var flags int
|
||||
if err := db.QueryRow("SELECT label,doc,flags+0,HEX(binary_def),slash FROM "+q(parent+"_copy")).Scan(&label, &doc, &flags, &binaryHex, &slash); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, table := range read().Schemas[0].Tables {
|
||||
if table.Name == parent+"_copy" && table.Metadata["mysql_auto_increment"] != "18446744073709551615" {
|
||||
t.Errorf("large auto-increment counter lost: %v", table.Metadata["mysql_auto_increment"])
|
||||
}
|
||||
}
|
||||
if label != "don't (change)" || doc != "first;\n-- data, not a comment\nlast" || flags != 10 || binaryHex != "00275CFF" || slash != `C:\temp\file` {
|
||||
t.Errorf("defaults corrupted: %q %q %d %s %q", label, doc, flags, binaryHex, slash)
|
||||
}
|
||||
}
|
||||
+339
-96
@@ -7,13 +7,14 @@ import (
|
||||
"io"
|
||||
"os"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
_ "github.com/go-sql-driver/mysql"
|
||||
|
||||
"git.warky.dev/wdevs/relspecgo/pkg/mariadb"
|
||||
"git.warky.dev/wdevs/relspecgo/pkg/models"
|
||||
"git.warky.dev/wdevs/relspecgo/pkg/transform"
|
||||
"git.warky.dev/wdevs/relspecgo/pkg/writers"
|
||||
_ "github.com/go-sql-driver/mysql"
|
||||
)
|
||||
|
||||
type Writer struct {
|
||||
@@ -22,88 +23,192 @@ type Writer struct {
|
||||
}
|
||||
|
||||
func NewWriter(options *writers.WriterOptions) *Writer { return &Writer{options: options} }
|
||||
func (w *Writer) WriteDatabase(db *models.Database) error {
|
||||
|
||||
// Statements preflights the whole model, emits conversion diagnostics, and returns
|
||||
// executable units. No output files or database connections are opened here.
|
||||
func (w *Writer) Statements(db *models.Database) ([]string, error) {
|
||||
if w.options == nil {
|
||||
return fmt.Errorf("writer options are required")
|
||||
return nil, fmt.Errorf("writer options are required")
|
||||
}
|
||||
if conn, ok := w.options.Metadata["connection_string"].(string); ok && conn != "" {
|
||||
return w.execute(db, conn)
|
||||
if db == nil {
|
||||
return nil, fmt.Errorf("database is required")
|
||||
}
|
||||
release, err := w.openOutput()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer release()
|
||||
return w.writeDatabaseDDL(db)
|
||||
}
|
||||
|
||||
func (w *Writer) writeDatabaseDDL(db *models.Database) error {
|
||||
if _, err := fmt.Fprintf(w.writer, "-- MySQL Database Schema\n-- Database: %s\n-- Generated by RelSpec\n\n", db.Name); err != nil {
|
||||
return err
|
||||
}
|
||||
for _, s := range db.Schemas {
|
||||
if err := w.WriteSchema(s); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// openOutput points w.writer at the configured destination (output file or
|
||||
// stdout) when none is set, and returns a func that releases it again.
|
||||
func (w *Writer) openOutput() (func(), error) {
|
||||
if w.writer != nil {
|
||||
return func() {}, nil
|
||||
}
|
||||
if w.options != nil && w.options.OutputPath != "" {
|
||||
f, err := os.Create(w.options.OutputPath)
|
||||
if w.options.LowercaseIdentifiers {
|
||||
var err error
|
||||
db, err = transform.LowercaseIdentifiers(db)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
w.writer = f
|
||||
return func() {
|
||||
f.Close()
|
||||
w.writer = nil
|
||||
}, nil
|
||||
}
|
||||
w.writer = os.Stdout
|
||||
return func() { w.writer = nil }, nil
|
||||
diagnostics := conversionDiagnostics(db)
|
||||
if len(diagnostics) > 0 && w.options.StrictConversion {
|
||||
return nil, fmt.Errorf("lossy MySQL conversion rejected:\n- %s", strings.Join(diagnostics, "\n- "))
|
||||
}
|
||||
for _, d := range diagnostics {
|
||||
if w.options.Warning != nil {
|
||||
w.options.Warning(d)
|
||||
} else {
|
||||
fmt.Fprintln(os.Stderr, "warning: MySQL conversion: "+d)
|
||||
}
|
||||
}
|
||||
return databaseStatements(db), nil
|
||||
}
|
||||
|
||||
func (w *Writer) WriteSchema(s *models.Schema) error {
|
||||
release, err := w.openOutput()
|
||||
func (w *Writer) WriteDatabase(db *models.Database) error {
|
||||
statements, err := w.Statements(db)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer release()
|
||||
for _, t := range s.Tables {
|
||||
if err := w.writeTable(s, t); err != nil {
|
||||
if conn, ok := w.options.Metadata["connection_string"].(string); ok && conn != "" {
|
||||
return w.execute(statements, conn)
|
||||
}
|
||||
if w.writer != nil {
|
||||
return writeStatements(w.writer, statements, db.Name)
|
||||
}
|
||||
if w.options.OutputPath == "" {
|
||||
return writeStatements(os.Stdout, statements, db.Name)
|
||||
}
|
||||
f, err := os.Create(w.options.OutputPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
err = writeStatements(f, statements, db.Name)
|
||||
closeErr := f.Close()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return closeErr
|
||||
}
|
||||
func writeStatements(out io.Writer, statements []string, name string) error {
|
||||
name = strings.NewReplacer("\n", " ", "\r", " ").Replace(name)
|
||||
if _, err := fmt.Fprintf(out, "-- Database: %s\n", name); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := io.WriteString(out, "-- MySQL table schema; generated by RelSpec\n\n"); err != nil {
|
||||
return err
|
||||
}
|
||||
for _, stmt := range statements {
|
||||
if _, err := io.WriteString(out, stmt); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (w *Writer) WriteSchema(s *models.Schema) error {
|
||||
return w.WriteDatabase(&models.Database{Schemas: []*models.Schema{s}})
|
||||
}
|
||||
func (w *Writer) WriteTable(t *models.Table) error {
|
||||
release, err := w.openOutput()
|
||||
if err != nil {
|
||||
return err
|
||||
return w.WriteSchema(&models.Schema{Name: "", Tables: []*models.Table{t}})
|
||||
}
|
||||
|
||||
func qualified(schema, name string) string {
|
||||
if schema == "" {
|
||||
return quote(name)
|
||||
}
|
||||
defer release()
|
||||
return w.writeTable(nil, t)
|
||||
return quote(schema) + "." + quote(name)
|
||||
}
|
||||
func keys[T any](m map[string]T) []string {
|
||||
out := make([]string, 0, len(m))
|
||||
for k := range m {
|
||||
out = append(out, k)
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
func metaString(m map[string]any, key string) string { v, _ := m[key].(string); return v }
|
||||
func metaBool(m map[string]any, key string, def bool) bool {
|
||||
v, ok := m[key].(bool)
|
||||
if ok {
|
||||
return v
|
||||
}
|
||||
return def
|
||||
}
|
||||
func signature(c *models.Column) string {
|
||||
return fmt.Sprintf("%s/%d/%d/%d", c.Type, c.Length, c.Precision, c.Scale)
|
||||
}
|
||||
func columnType(c *models.Column) string {
|
||||
if native := metaString(c.Metadata, "mysql_type"); native != "" && metaString(c.Metadata, "mysql_type_signature") == signature(c) {
|
||||
return native
|
||||
}
|
||||
typ := mariadb.ConvertCanonicalToMariaDB(c.Type)
|
||||
switch strings.ToLower(c.Type) {
|
||||
case "string":
|
||||
if c.Length > 0 {
|
||||
typ = fmt.Sprintf("VARCHAR(%d)", c.Length)
|
||||
}
|
||||
case "decimal":
|
||||
if c.Precision > 0 {
|
||||
typ = fmt.Sprintf("DECIMAL(%d,%d)", c.Precision, c.Scale)
|
||||
}
|
||||
case "timestamp", "time":
|
||||
if c.Precision > 0 {
|
||||
typ += fmt.Sprintf("(%d)", c.Precision)
|
||||
}
|
||||
}
|
||||
return typ
|
||||
}
|
||||
|
||||
func (w *Writer) writeTable(s *models.Schema, t *models.Table) error {
|
||||
_, err := io.WriteString(w.writer, tableStatement(s, t))
|
||||
return err
|
||||
// MySQL string literals use doubled quotes and escaped backslashes. The output
|
||||
// assumes the server's standard SQL mode (NO_BACKSLASH_ESCAPES is not enabled).
|
||||
func literal(s string) string {
|
||||
return "'" + strings.ReplaceAll(strings.ReplaceAll(s, "\\", "\\\\"), "'", "''") + "'"
|
||||
}
|
||||
func defaultSQL(c *models.Column) string {
|
||||
value := fmt.Sprint(c.Default)
|
||||
if native := metaString(c.Metadata, "mysql_default_sql"); native != "" && metaString(c.Metadata, "mysql_default_value") == value {
|
||||
return native
|
||||
}
|
||||
if strings.HasPrefix(metaString(c.Metadata, "mysql_type"), "bit(") && strings.HasPrefix(value, "b'") && strings.HasSuffix(value, "'") && strings.Trim(value[2:len(value)-1], "01") == "" {
|
||||
return value
|
||||
}
|
||||
if metaBool(c.Metadata, "mysql_default_expression", false) {
|
||||
if strings.HasPrefix(strings.ToUpper(value), "CURRENT_TIMESTAMP") {
|
||||
return value
|
||||
}
|
||||
return "(" + value + ")"
|
||||
}
|
||||
switch strings.ToLower(c.Type) {
|
||||
case "text", "json", "jsonb", "bytea":
|
||||
return "(" + literal(value) + ")"
|
||||
case "string", "uuid", "date", "time", "timestamp", "timestamptz":
|
||||
return literal(value)
|
||||
case "bool", "boolean":
|
||||
if value == "true" {
|
||||
return "1"
|
||||
}
|
||||
if value == "false" {
|
||||
return "0"
|
||||
}
|
||||
}
|
||||
// Numeric defaults are values, never executable fragments.
|
||||
if _, err := strconv.ParseFloat(value, 64); err == nil {
|
||||
return value
|
||||
}
|
||||
return literal(value)
|
||||
}
|
||||
func indexParts(i *models.Index) string {
|
||||
if len(i.KeyParts) == 0 {
|
||||
return quoted(i.Columns)
|
||||
}
|
||||
parts := make([]string, 0, len(i.KeyParts))
|
||||
for _, p := range i.KeyParts {
|
||||
s := quote(p.Name)
|
||||
if p.Expression != "" {
|
||||
s = "(" + p.Expression + ")"
|
||||
}
|
||||
if p.Length > 0 {
|
||||
s += fmt.Sprintf("(%d)", p.Length)
|
||||
}
|
||||
if p.Descending {
|
||||
s += " DESC"
|
||||
}
|
||||
parts = append(parts, s)
|
||||
}
|
||||
return strings.Join(parts, ", ")
|
||||
}
|
||||
|
||||
func tableStatement(s *models.Schema, t *models.Table) string {
|
||||
name := t.Name
|
||||
schema := ""
|
||||
if s != nil {
|
||||
name = fmt.Sprintf("%s.%s", quote(s.Name), quote(t.Name))
|
||||
} else {
|
||||
name = quote(name)
|
||||
schema = s.Name
|
||||
}
|
||||
cols := make([]*models.Column, 0, len(t.Columns))
|
||||
for _, c := range t.Columns {
|
||||
@@ -118,76 +223,196 @@ func tableStatement(s *models.Schema, t *models.Table) string {
|
||||
defs := []string{}
|
||||
pk := []string{}
|
||||
for _, c := range cols {
|
||||
d := fmt.Sprintf(" %s %s", quote(c.Name), mariadb.ConvertCanonicalToMariaDB(c.Type))
|
||||
if c.Length > 0 && strings.EqualFold(c.Type, "string") {
|
||||
d = fmt.Sprintf(" %s VARCHAR(%d)", quote(c.Name), c.Length)
|
||||
d := " " + quote(c.Name) + " " + columnType(c)
|
||||
if charset := metaString(c.Metadata, "mysql_charset"); charset != "" && c.Collation != metaString(t.Metadata, "mysql_collation") {
|
||||
d += " CHARACTER SET " + quote(charset)
|
||||
}
|
||||
if c.Collation != "" && c.Collation != metaString(t.Metadata, "mysql_collation") {
|
||||
d += " COLLATE " + quote(c.Collation)
|
||||
}
|
||||
if c.Generated {
|
||||
d += " GENERATED ALWAYS AS (" + c.GenerationExpression + ")"
|
||||
if strings.EqualFold(metaString(c.Metadata, "mysql_generation_storage"), "STORED") {
|
||||
d += " STORED"
|
||||
} else {
|
||||
d += " VIRTUAL"
|
||||
}
|
||||
}
|
||||
if c.NotNull {
|
||||
d += " NOT NULL"
|
||||
} else if !c.Generated {
|
||||
d += " NULL"
|
||||
}
|
||||
if c.AutoIncrement {
|
||||
d += " AUTO_INCREMENT"
|
||||
}
|
||||
if c.Default != nil {
|
||||
d += fmt.Sprintf(" DEFAULT %s", writers.QuoteDefaultValue(fmt.Sprint(c.Default), c.Type))
|
||||
if c.Default != nil && !c.Generated {
|
||||
d += " DEFAULT " + defaultSQL(c)
|
||||
}
|
||||
if upd := metaString(c.Metadata, "mysql_on_update"); upd != "" {
|
||||
d += " ON UPDATE " + upd
|
||||
}
|
||||
if metaBool(c.Metadata, "mysql_invisible", false) {
|
||||
d += " INVISIBLE"
|
||||
}
|
||||
if c.Comment != "" {
|
||||
d += " COMMENT " + literal(c.Comment)
|
||||
}
|
||||
defs = append(defs, d)
|
||||
if c.IsPrimaryKey {
|
||||
pk = append(pk, quote(c.Name))
|
||||
pk = append(pk, c.Name)
|
||||
}
|
||||
}
|
||||
constraintNames := make([]string, 0, len(t.Constraints))
|
||||
for name := range t.Constraints {
|
||||
constraintNames = append(constraintNames, name)
|
||||
}
|
||||
sort.Strings(constraintNames)
|
||||
for _, name := range constraintNames {
|
||||
for _, name := range keys(t.Constraints) {
|
||||
c := t.Constraints[name]
|
||||
if c.Type == models.PrimaryKeyConstraint {
|
||||
pk = nil
|
||||
for _, n := range c.Columns {
|
||||
pk = append(pk, quote(n))
|
||||
}
|
||||
pk = c.Columns
|
||||
break
|
||||
}
|
||||
}
|
||||
if len(pk) > 0 {
|
||||
defs = append(defs, " PRIMARY KEY ("+strings.Join(pk, ", ")+")")
|
||||
}
|
||||
for _, name := range constraintNames {
|
||||
c := t.Constraints[name]
|
||||
if c.Type == models.UniqueConstraint {
|
||||
defs = append(defs, fmt.Sprintf(" CONSTRAINT %s UNIQUE (%s)", quote(c.Name), quoted(c.Columns)))
|
||||
var primaryIndex *models.Index
|
||||
for name, i := range t.Indexes {
|
||||
if strings.EqualFold(name, "PRIMARY") {
|
||||
primaryIndex = i
|
||||
break
|
||||
}
|
||||
}
|
||||
return fmt.Sprintf("CREATE TABLE IF NOT EXISTS %s (\n%s\n) ENGINE=InnoDB;\n\n", name, strings.Join(defs, ",\n"))
|
||||
if i := primaryIndex; i != nil {
|
||||
defs = append(defs, " PRIMARY KEY ("+indexParts(i)+")")
|
||||
} else if len(pk) > 0 {
|
||||
defs = append(defs, " PRIMARY KEY ("+quoted(pk)+")")
|
||||
}
|
||||
for _, name := range keys(t.Constraints) {
|
||||
c := t.Constraints[name]
|
||||
switch c.Type {
|
||||
case models.UniqueConstraint:
|
||||
if t.Indexes[c.Name] == nil {
|
||||
defs = append(defs, " CONSTRAINT "+quote(c.Name)+" UNIQUE ("+quoted(c.Columns)+")")
|
||||
}
|
||||
case models.CheckConstraint:
|
||||
d := " CONSTRAINT " + quote(c.Name) + " CHECK (" + c.Expression + ")"
|
||||
if !metaBool(c.Metadata, "mysql_enforced", true) {
|
||||
d += " NOT ENFORCED"
|
||||
}
|
||||
defs = append(defs, d)
|
||||
}
|
||||
}
|
||||
for _, name := range keys(t.Indexes) {
|
||||
i := t.Indexes[name]
|
||||
if strings.EqualFold(name, "PRIMARY") {
|
||||
continue
|
||||
}
|
||||
method := strings.ToUpper(i.Type)
|
||||
if method != "" && method != "BTREE" && method != "HASH" && method != "FULLTEXT" && method != "SPATIAL" {
|
||||
continue
|
||||
}
|
||||
kind := "KEY"
|
||||
if i.Unique {
|
||||
kind = "UNIQUE KEY"
|
||||
}
|
||||
if method == "FULLTEXT" || method == "SPATIAL" {
|
||||
kind = method + " KEY"
|
||||
}
|
||||
d := " " + kind + " " + quote(i.Name) + " (" + indexParts(i) + ")"
|
||||
if method == "HASH" {
|
||||
d += " USING " + method
|
||||
}
|
||||
if !metaBool(i.Metadata, "mysql_visible", true) {
|
||||
d += " INVISIBLE"
|
||||
}
|
||||
if i.Comment != "" {
|
||||
d += " COMMENT " + literal(i.Comment)
|
||||
}
|
||||
defs = append(defs, d)
|
||||
}
|
||||
engine := metaString(t.Metadata, "mysql_engine")
|
||||
if engine == "" {
|
||||
engine = "InnoDB"
|
||||
}
|
||||
tail := " ENGINE=" + quote(engine)
|
||||
if engine == "InnoDB" {
|
||||
tail = " ENGINE=InnoDB"
|
||||
}
|
||||
if coll := metaString(t.Metadata, "mysql_collation"); coll != "" {
|
||||
tail += " COLLATE=" + quote(coll)
|
||||
}
|
||||
if auto := metaString(t.Metadata, "mysql_auto_increment"); auto != "" {
|
||||
if _, err := strconv.ParseUint(auto, 10, 64); err == nil {
|
||||
tail += " AUTO_INCREMENT=" + auto
|
||||
}
|
||||
}
|
||||
if t.Comment != "" {
|
||||
tail += " COMMENT=" + literal(t.Comment)
|
||||
}
|
||||
return "CREATE TABLE IF NOT EXISTS " + qualified(schema, t.Name) + " (\n" + strings.Join(defs, ",\n") + "\n)" + tail + ";\n\n"
|
||||
}
|
||||
|
||||
func databaseStatements(db *models.Database) []string {
|
||||
var statements []string
|
||||
for _, schema := range db.Schemas {
|
||||
for _, table := range schema.Tables {
|
||||
statements = append(statements, tableStatement(schema, table))
|
||||
for _, s := range db.Schemas {
|
||||
for _, t := range s.Tables {
|
||||
statements = append(statements, tableStatement(s, t))
|
||||
}
|
||||
}
|
||||
// References are added after all tables, supporting cycles and forward references.
|
||||
for _, s := range db.Schemas {
|
||||
for _, t := range s.Tables {
|
||||
for _, name := range keys(t.Constraints) {
|
||||
c := t.Constraints[name]
|
||||
if c.Type != models.ForeignKeyConstraint {
|
||||
continue
|
||||
}
|
||||
rs := c.ReferencedSchema
|
||||
if rs == "" {
|
||||
rs = s.Name
|
||||
}
|
||||
stmt := "ALTER TABLE " + qualified(s.Name, t.Name) + " ADD CONSTRAINT " + quote(c.Name) + " FOREIGN KEY (" + quoted(c.Columns) + ") REFERENCES " + qualified(rs, c.ReferencedTable) + " (" + quoted(c.ReferencedColumns) + ")"
|
||||
if validAction(c.OnDelete) && c.OnDelete != "" {
|
||||
stmt += " ON DELETE " + strings.ToUpper(c.OnDelete)
|
||||
}
|
||||
if validAction(c.OnUpdate) && c.OnUpdate != "" {
|
||||
stmt += " ON UPDATE " + strings.ToUpper(c.OnUpdate)
|
||||
}
|
||||
statements = append(statements, stmt+";\n\n")
|
||||
}
|
||||
}
|
||||
}
|
||||
return statements
|
||||
}
|
||||
|
||||
func (w *Writer) execute(dbm *models.Database, conn string) error {
|
||||
statements := databaseStatements(dbm)
|
||||
func (w *Writer) execute(statements []string, conn string) error {
|
||||
if w.options.AtomicExecution {
|
||||
return fmt.Errorf("MySQL DDL implicitly commits; atomic execution is unsupported")
|
||||
}
|
||||
db, err := sql.Open("mysql", conn)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to connect: %w", err)
|
||||
}
|
||||
defer db.Close()
|
||||
for _, stmt := range statements {
|
||||
if _, err := db.ExecContext(context.Background(), stmt); err != nil {
|
||||
return fmt.Errorf("failed to execute SQL: %w", err)
|
||||
session, err := db.Conn(context.Background())
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to execute SQL: %w", err)
|
||||
}
|
||||
defer session.Close()
|
||||
var mode string
|
||||
if err := session.QueryRowContext(context.Background(), "SELECT @@SESSION.sql_mode").Scan(&mode); err != nil {
|
||||
return fmt.Errorf("read MySQL SQL mode: %w", err)
|
||||
}
|
||||
if mode = unsupportedMode(mode); mode != "" {
|
||||
return fmt.Errorf("MySQL execution requires standard quotes and backslash escaping; %s is enabled", mode)
|
||||
}
|
||||
var failures []string
|
||||
for n, stmt := range statements {
|
||||
if _, err := session.ExecContext(context.Background(), stmt); err != nil {
|
||||
failures = append(failures, fmt.Sprintf("statement %d: %v", n+1, err))
|
||||
if !w.options.ContinueOnError {
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
if len(failures) > 0 {
|
||||
return fmt.Errorf("failed to execute SQL: %s", strings.Join(failures, "; "))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func quote(s string) string { return "`" + strings.ReplaceAll(s, "`", "``") + "`" }
|
||||
func quoted(xs []string) string {
|
||||
out := make([]string, len(xs))
|
||||
@@ -196,3 +421,21 @@ func quoted(xs []string) string {
|
||||
}
|
||||
return strings.Join(out, ", ")
|
||||
}
|
||||
|
||||
func validAction(action string) bool {
|
||||
switch strings.ToUpper(action) {
|
||||
case "", "CASCADE", "SET NULL", "RESTRICT", "NO ACTION":
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func unsupportedMode(mode string) string {
|
||||
var modes []string
|
||||
for _, unsupported := range []string{"NO_BACKSLASH_ESCAPES", "ANSI_QUOTES"} {
|
||||
if strings.Contains(strings.ToUpper(mode), unsupported) {
|
||||
modes = append(modes, unsupported)
|
||||
}
|
||||
}
|
||||
return strings.Join(modes, ", ")
|
||||
}
|
||||
|
||||
@@ -56,6 +56,11 @@ const (
|
||||
|
||||
// WriterOptions contains common options for writers
|
||||
type WriterOptions struct {
|
||||
// StrictConversion rejects known lossy MySQL conversions before output or execution.
|
||||
StrictConversion bool
|
||||
// Warning receives conversion diagnostics. MySQL writes warnings to stderr when nil.
|
||||
Warning func(string)
|
||||
|
||||
// OutputPath is the path where the output should be written
|
||||
OutputPath string
|
||||
|
||||
|
||||
+18
-1
@@ -22,7 +22,7 @@ Podman (preferred) or Docker. One tool for postgres, mssql, mysql.
|
||||
|---|---|---|---|---|
|
||||
| postgres | 5439 | `RELSPEC_TEST_PG_CONN` | `tests/postgres/init.sql` | readers/pgsql, writers/pgsql, writers/sqlexec |
|
||||
| mssql | 1439 | `RELSPEC_TEST_MSSQL_CONN` | `test_data/mssql/test_schema.sql` (creates `RelSpecTest`) | readers/mssql, writers/mssql |
|
||||
| mysql | 3309 | `RELSPEC_TEST_MYSQL_CONN` | `tests/dbtest/init/mysql.sql` | none (no Go driver/reader yet) |
|
||||
| mysql | 3309 | `RELSPEC_TEST_MYSQL_CONN` | `tests/dbtest/init/mysql.sql` | readers/mysql, writers/mysql |
|
||||
|
||||
## Env
|
||||
|
||||
@@ -43,3 +43,20 @@ Podman (preferred) or Docker. One tool for postgres, mssql, mysql.
|
||||
- Containers are named `relspec-test-<db>`; existing `tests/postgres/*.sh` and `make docker-*` use the same postgres name/port 5439 and remain independent.
|
||||
- mssql needs ~2GB RAM and takes longer to become ready.
|
||||
- Tests skip when the env var is unset.
|
||||
|
||||
## MySQL schema fidelity tests
|
||||
|
||||
Run `./tests/dbtest/dbtest.sh test mysql` for the MySQL 8.4 round-trip and
|
||||
unsupported-object regressions. They use temporary tables in `relspec_test`,
|
||||
including checks, functional indexes, composite foreign keys, binary/expression
|
||||
defaults, and generated columns. The test account needs table, view, and trigger
|
||||
privileges in that database. The disposable runner enables
|
||||
`log_bin_trust_function_creators` so binary logging does not require SUPER for
|
||||
trigger fixtures. For a separately managed test server, disable binary logging
|
||||
or configure that setting first. No global database-creation privilege is required.
|
||||
A failed or interrupted test can leave tables named `rt_*` or `unsupported_*`;
|
||||
restarting the disposable container removes them.
|
||||
|
||||
See [the MySQL capability matrix](../../docs/MYSQL_FIDELITY.md) for the supported
|
||||
contract and strict conversion options. PostgreSQL/SQL Server tests remain
|
||||
separate; this suite does not certify MariaDB compatibility.
|
||||
|
||||
@@ -8,9 +8,14 @@ DB_ENV=(-e MYSQL_ROOT_PASSWORD=relspec_root_password -e MYSQL_DATABASE=relspec_t
|
||||
DB_INIT_MOUNT="$ROOT/tests/dbtest/init/mysql.sql:/docker-entrypoint-initdb.d/init.sql"
|
||||
DB_CONN_VAR=RELSPEC_TEST_MYSQL_CONN
|
||||
DB_CONN="relspec:relspec_test_password@tcp(localhost:3309)/relspec_test"
|
||||
DB_DEFAULT_PKGS=""
|
||||
DB_DEFAULT_PKGS="./pkg/readers/mysql ./pkg/writers/mysql"
|
||||
|
||||
db_ready() { rt exec "$DB_CONTAINER" mysqladmin ping -h 127.0.0.1 -urelspec -prelspec_test_password --silent >/dev/null 2>&1; }
|
||||
db_post_init() { :; }
|
||||
# Configure the final server after readiness; SET GLOBAL during entrypoint init
|
||||
# only affects its temporary server and is reset before the tests connect.
|
||||
db_post_init() {
|
||||
rt exec "$DB_CONTAINER" mysql -uroot -prelspec_root_password \
|
||||
-e 'SET GLOBAL log_bin_trust_function_creators = 1' >/dev/null
|
||||
}
|
||||
db_shell() { rt exec -it "$DB_CONTAINER" mysql -urelspec -prelspec_test_password relspec_test "$@"; }
|
||||
db_exec_file() { rt exec -i "$DB_CONTAINER" mysql -urelspec -prelspec_test_password relspec_test < "$1"; }
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
-- Minimal MySQL fixture for relspec container tests.
|
||||
-- Extend when a MySQL reader/writer is added.
|
||||
-- Reader/writer live tests create additional temporary fixtures.
|
||||
CREATE TABLE users (
|
||||
id BIGINT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY,
|
||||
email VARCHAR(255) NOT NULL,
|
||||
|
||||
Reference in New Issue
Block a user