fix: preserve MySQL schema fidelity

This commit is contained in:
2026-10-12 21:56:48 +02:00
parent dd97413b8e
commit 05466a92b1
26 changed files with 1869 additions and 169 deletions
+1
View File
@@ -277,6 +277,7 @@ func parseHumanSize(s string) (int64, error) {
// Options carries the subset of command flags a job file may set.
type Options struct {
StrictConversion bool `yaml:"strict_conversion"`
LowercaseIdentifiers bool `yaml:"lowercase_identifiers"`
FlattenSchema bool `yaml:"flatten_schema"`
Schema string `yaml:"schema"`
+19 -19
View File
@@ -61,6 +61,7 @@ var MariaDBToCanonicalTypes = map[string]string{
// CanonicalToMariaDBTypes maps canonical types to MariaDB/MySQL types.
var CanonicalToMariaDBTypes = map[string]string{
"bool": "TINYINT(1)",
"boolean": "TINYINT(1)",
"int8": "TINYINT",
"int16": "SMALLINT",
"int": "INT",
@@ -114,20 +115,26 @@ func NormalizeMariaDBType(baseType string) string {
// ConvertMariaDBToCanonical converts a MariaDB/MySQL type name to the canonical type.
// Strips dimension parameters and normalizes aliases. Defaults to "string".
func ConvertMariaDBToCanonical(mariadbType string) string {
base := strings.ToLower(strings.TrimSpace(mariadbType))
if idx := strings.Index(base, "("); idx >= 0 {
base = strings.TrimSpace(base[:idx])
lower := strings.ToLower(strings.TrimSpace(mariadbType))
base := lower
if idx := strings.IndexAny(base, "( "); idx >= 0 {
base = base[:idx]
}
if canonical, ok := MariaDBToCanonicalTypes[base]; ok {
return canonical
}
// Prefix match for composite types (e.g., "unsigned bigint")
for key, canonical := range MariaDBToCanonicalTypes {
if strings.HasPrefix(base, key) {
return canonical
canonical, ok := MariaDBToCanonicalTypes[base]
if ok {
if strings.Contains(lower, "unsigned") || strings.Contains(lower, "zerofill") {
switch canonical {
case "int8":
return "uint8"
case "int16":
return "uint16"
case "int":
return "uint"
case "int64":
return "uint64"
}
}
return canonical
}
return "string"
@@ -145,12 +152,5 @@ func ConvertCanonicalToMariaDB(canonicalType string) string {
return mariadbType
}
// Prefix fallback
for canonical, mariadb := range CanonicalToMariaDBTypes {
if strings.HasPrefix(lower, canonical) {
return mariadb
}
}
return "VARCHAR(255)"
}
+13
View File
@@ -0,0 +1,13 @@
package mariadb
import "testing"
func TestIntegerSignedness(t *testing.T) {
for _, tc := range []struct{ input, want string }{
{"tinyint(3) unsigned", "uint8"}, {"smallint unsigned", "uint16"}, {"mediumint unsigned", "uint"}, {"int unsigned", "uint"}, {"bigint unsigned", "uint64"}, {"bigint zerofill", "uint64"}, {"bigint", "int64"}, {"varchar(32)", "string"},
} {
if got := ConvertMariaDBToCanonical(tc.input); got != tc.want {
t.Errorf("%s: got %s want %s", tc.input, got, tc.want)
}
}
}
+11
View File
@@ -256,9 +256,19 @@ func (d *Column) SQLName() string {
return strings.ToLower(d.Name)
}
// IndexPart is an ordered MySQL index key part. Name and Expression are mutually
// exclusive; expressions are executable SQL and are not identifier-normalized.
type IndexPart struct {
Name string `json:"name,omitempty" yaml:"name,omitempty" xml:"name,omitempty"`
Expression string `json:"expression,omitempty" yaml:"expression,omitempty" xml:"expression,omitempty"`
Length int `json:"length,omitempty" yaml:"length,omitempty" xml:"length,omitempty"`
Descending bool `json:"descending,omitempty" yaml:"descending,omitempty" xml:"descending,omitempty"`
}
// Index represents a database index for optimizing query performance.
// Indexes can be unique, partial, or include additional columns.
type Index struct {
KeyParts []IndexPart `json:"key_parts,omitempty" yaml:"key_parts,omitempty" xml:"key_parts,omitempty"`
Name string `json:"name" yaml:"name" xml:"name"`
Description string `json:"description,omitempty" yaml:"description,omitempty" xml:"description,omitempty"`
Table string `json:"table,omitempty" yaml:"table,omitempty" xml:"table,omitempty"`
@@ -318,6 +328,7 @@ func (d *Relationship) SQLName() string {
// Constraint represents a database constraint that enforces data integrity rules.
// Constraints can be primary keys, foreign keys, unique constraints, check constraints, or not-null constraints.
type Constraint struct {
Metadata map[string]any `json:"metadata,omitempty" yaml:"metadata,omitempty" xml:"-"`
Name string `json:"name" yaml:"name" xml:"name"`
Type ConstraintType `json:"type" yaml:"type" xml:"type"`
Columns []string `json:"columns" yaml:"columns" xml:"columns"`
+194
View File
@@ -0,0 +1,194 @@
package mysql
import (
"regexp"
"strings"
)
// columnDefaults extracts native DEFAULT expressions from SHOW CREATE TABLE.
// MySQL can truncate binary defaults in information_schema (for example X'00FF'
// appears as just "0x"). SHOW CREATE is authoritative for their SQL spelling.
// Separators and whitespace inside quoted strings, identifiers and nested
// expressions are never treated as definition/default boundaries.
func columnDefaults(create string) map[string]string {
out := map[string]string{}
depth, start := 0, 0
var quoted byte
definitions := []string{}
for i := 0; i < len(create); i++ {
ch := create[i]
if quoted != 0 {
if ch == '\\' && quoted != '`' && i+1 < len(create) {
i++
continue
}
if ch == quoted {
if i+1 < len(create) && create[i+1] == quoted {
i++
} else {
quoted = 0
}
}
continue
}
if ch == '\'' || ch == '"' || ch == '`' {
quoted = ch
continue
}
switch ch {
case '(':
depth++
if depth == 1 {
start = i + 1
}
case ',':
if depth == 1 {
definitions = append(definitions, create[start:i])
start = i + 1
}
case ')':
depth--
if depth == 0 {
definitions = append(definitions, create[start:i])
break
}
}
if ch == ')' && depth == 0 {
break
}
}
for _, def := range definitions {
def = strings.TrimSpace(def)
if len(def) == 0 || def[0] != '`' {
continue
}
var name strings.Builder
end := 1
for end < len(def) {
if def[end] == '`' {
if end+1 < len(def) && def[end+1] == '`' {
name.WriteByte('`')
end += 2
continue
}
end++
break
}
name.WriteByte(def[end])
end++
}
// Consume full tokens so DEFAULT inside a generated expression or literal
// cannot masquerade as a column default.
for end < len(def) {
for end < len(def) && isSpace(def[end]) {
end++
}
begin := end
end = tokenEnd(def, end)
if end == begin {
break
}
if strings.EqualFold(def[begin:end], "DEFAULT") {
for end < len(def) && isSpace(def[end]) {
end++
}
finish := tokenEnd(def, end)
out[name.String()] = def[end:finish]
break
}
}
}
return out
}
func isSpace(b byte) bool { return b == ' ' || b == '\n' || b == '\r' || b == '\t' }
func tokenEnd(s string, start int) int {
depth := 0
var quoted byte
for i := start; i < len(s); i++ {
ch := s[i]
if quoted != 0 {
if ch == '\\' && quoted != '`' && i+1 < len(s) {
i++
continue
}
if ch == quoted {
if i+1 < len(s) && s[i+1] == quoted {
i++
} else {
quoted = 0
}
}
continue
}
if ch == '\'' || ch == '"' || ch == '`' {
quoted = ch
continue
}
if ch == '(' {
depth++
}
if ch == ')' {
depth--
}
if depth == 0 && isSpace(ch) {
return i
}
}
return len(s)
}
// SHOW CREATE has native options not exposed by STATISTICS/COLUMNS. Ignore
// quoted identifiers and values when detecting options outside our contract.
func unquotedSQL(create string) string {
var code strings.Builder
var quoted byte
for i := 0; i < len(create); i++ {
ch := create[i]
if quoted != 0 {
if ch == '\\' && quoted != '`' && i+1 < len(create) {
i++
continue
}
if ch == quoted {
if i+1 < len(create) && create[i+1] == quoted {
i++
} else {
quoted = 0
code.WriteByte(' ')
}
}
continue
}
if ch == '\'' || ch == '"' || ch == '`' {
quoted = ch
code.WriteByte(' ')
continue
}
code.WriteByte(ch)
}
return code.String()
}
var autoIncrementOption = regexp.MustCompile(`(?i)\bAUTO_INCREMENT\s*=\s*([0-9]+)`)
func autoIncrementValue(create string) string {
if match := autoIncrementOption.FindStringSubmatch(unquotedSQL(create)); len(match) > 1 {
return match[1]
}
return "1"
}
func unsupportedNativeOptions(create string) []string {
words := strings.FieldsFunc(strings.ToUpper(unquotedSQL(create)), func(r rune) bool { return !(r >= 'A' && r <= 'Z' || r == '_') })
var out []string
for i, w := range words {
switch w {
case "ENGINE_ATTRIBUTE", "SECONDARY_ENGINE_ATTRIBUTE", "KEY_BLOCK_SIZE", "COLUMN_FORMAT", "STORAGE":
out = append(out, w)
case "PARSER":
if i > 0 && words[i-1] == "WITH" {
out = append(out, "WITH PARSER")
}
}
}
return out
}
+54
View File
@@ -0,0 +1,54 @@
package mysql
import "testing"
func TestUnescapeExpression(t *testing.T) {
for _, tc := range []struct{ input, want string }{
{`_utf8mb4\'one\\ntwo\'`, `_utf8mb4'one\ntwo'`},
{`concat(\'don\\\'t\',\' change\')`, `concat('don\'t',' change')`},
{`(uuid())`, `(uuid())`},
} {
if got := unescapeExpression(tc.input); got != tc.want {
t.Errorf("%q: got %q want %q", tc.input, got, tc.want)
}
}
}
func TestNativeColumnDefaults(t *testing.T) {
create := "CREATE TABLE `odd(table` (\n" +
" `a,b` varbinary(8) DEFAULT 0x0027FF,\n" +
" `tick``name` varchar(30) DEFAULT 'don''t, (split)',\n" +
" `expr` varchar(30) DEFAULT (concat('DEFAULT, ', (uuid()))),\n" +
" `generated` varchar(30) GENERATED ALWAYS AS (concat('DEFAULT',`expr`)) VIRTUAL,\n" +
" `new\nline` int DEFAULT 3,\n" +
" PRIMARY KEY (`a,b`)\n) ENGINE=InnoDB"
got := columnDefaults(create)
want := map[string]string{"a,b": "0x0027FF", "tick`name": "'don''t, (split)'", "expr": "(concat('DEFAULT, ', (uuid())))", "new\nline": "3"}
if len(got) != len(want) {
t.Fatalf("defaults: %v", got)
}
for name, value := range want {
if got[name] != value {
t.Errorf("%s: got %q want %q", name, got[name], value)
}
}
}
func TestNativeOptionsIgnoreQuotedValues(t *testing.T) {
create := "CREATE TABLE `WITH PARSER` (`STORAGE` int COMMENT 'KEY_BLOCK_SIZE', KEY `ENGINE_ATTRIBUTE` (`STORAGE`) WITH PARSER `ngram`) ENGINE=InnoDB"
options := unsupportedNativeOptions(create)
if len(options) != 1 || options[0] != "WITH PARSER" {
t.Fatalf("native options: %v", options)
}
}
func TestNativeAutoIncrementIgnoresQuotedValues(t *testing.T) {
for _, tc := range []struct{ create, want string }{
{"CREATE TABLE `x` (`id` bigint AUTO_INCREMENT) ENGINE=InnoDB AUTO_INCREMENT=18446744073709551615", "18446744073709551615"},
{"CREATE TABLE `x` (`id` bigint AUTO_INCREMENT) COMMENT='AUTO_INCREMENT=12'", "1"},
} {
if got := autoIncrementValue(tc.create); got != tc.want {
t.Errorf("got %s want %s", got, tc.want)
}
}
}
+252 -43
View File
@@ -4,9 +4,10 @@ import (
"context"
"database/sql"
"fmt"
"regexp"
"strings"
_ "github.com/go-sql-driver/mysql"
mysqlDriver "github.com/go-sql-driver/mysql"
"git.warky.dev/wdevs/relspecgo/pkg/mariadb"
"git.warky.dev/wdevs/relspecgo/pkg/models"
@@ -31,12 +32,13 @@ func (r *Reader) ReadDatabase() (*models.Database, error) {
return nil, fmt.Errorf("failed to connect: %w", err)
}
defer r.close()
var name, version string
if err := r.db.QueryRowContext(r.ctx, "SELECT DATABASE()").Scan(&name); err != nil {
var name, version, sqlMode string
if err := r.db.QueryRowContext(r.ctx, "SELECT DATABASE(), @@SESSION.sql_mode").Scan(&name, &sqlMode); err != nil {
return nil, fmt.Errorf("failed to get database name: %w", err)
}
_ = r.db.QueryRowContext(r.ctx, "SELECT VERSION()").Scan(&version)
db := models.InitDatabase(name)
db.Metadata["mysql_sql_mode"] = sqlMode
db.DatabaseType, db.SourceFormat, db.DatabaseVersion = models.MySQLDatabaseType, "mysql", version
schemas, err := r.querySchemas(name)
if err != nil {
@@ -48,8 +50,10 @@ func (r *Reader) ReadDatabase() (*models.Database, error) {
return nil, err
}
schema.Tables = tables
schema.Metadata["mysql_sql_mode"] = sqlMode
for _, table := range tables {
table.Columns, err = r.queryColumns(schema.Name, table.Name)
table.Metadata["mysql_sql_mode"] = sqlMode
table.Columns, err = r.queryColumns(schema.Name, table)
if err != nil {
return nil, err
}
@@ -62,6 +66,15 @@ func (r *Reader) ReadDatabase() (*models.Database, error) {
return nil, err
}
table.RefSchema = schema
for _, c := range table.Constraints {
if c.Type == models.PrimaryKeyConstraint {
for _, n := range c.Columns {
if col := table.Columns[n]; col != nil {
col.IsPrimaryKey = true
}
}
}
}
for _, c := range table.Constraints {
if c.Type == models.ForeignKeyConstraint {
r.deriveRelationship(table, c)
@@ -71,6 +84,9 @@ func (r *Reader) ReadDatabase() (*models.Database, error) {
schema.RefDatabase = db
db.Schemas = append(db.Schemas, schema)
}
if err := r.readUnsupported(db, name); err != nil {
return nil, err
}
return db, nil
}
@@ -97,7 +113,17 @@ func (r *Reader) ReadTable() (*models.Table, error) {
}
func (r *Reader) connect() error {
db, err := sql.Open("mysql", r.options.ConnectionString)
cfg, err := mysqlDriver.ParseDSN(r.options.ConnectionString)
if err != nil {
return err
}
if cfg.Params == nil {
cfg.Params = map[string]string{}
}
// SHOW CREATE parsing requires quoted column identifiers, including when the
// caller's metadata connection would otherwise disable show-create quoting.
cfg.Params["sql_quote_show_create"] = "1"
db, err := sql.Open("mysql", cfg.FormatDSN())
if err != nil {
return err
}
@@ -134,61 +160,147 @@ func (r *Reader) querySchemas(current string) ([]*models.Schema, error) {
}
func (r *Reader) queryTables(schema string) ([]*models.Table, error) {
rows, err := r.db.QueryContext(r.ctx, "SELECT TABLE_NAME FROM information_schema.TABLES WHERE TABLE_SCHEMA = ? AND TABLE_TYPE = 'BASE TABLE' ORDER BY TABLE_NAME", schema)
rows, err := r.db.QueryContext(r.ctx, "SELECT TABLE_NAME, ENGINE, TABLE_COLLATION, TABLE_COMMENT, CREATE_OPTIONS, AUTO_INCREMENT FROM information_schema.TABLES WHERE TABLE_SCHEMA = ? AND TABLE_TYPE = 'BASE TABLE' ORDER BY TABLE_NAME", schema)
if err != nil {
return nil, err
}
defer rows.Close()
var out []*models.Table
for rows.Next() {
var n string
if err := rows.Scan(&n); err != nil {
var n, engine, collation, comment, options string
var auto sql.NullString
if err := rows.Scan(&n, &engine, &collation, &comment, &options, &auto); err != nil {
return nil, err
}
out = append(out, models.InitTable(n, schema))
t := models.InitTable(n, schema)
t.Comment = comment
t.Metadata["mysql_engine"], t.Metadata["mysql_collation"], t.Metadata["mysql_create_options"] = engine, collation, options
if auto.Valid {
t.Metadata["mysql_auto_increment"] = auto.String
}
out = append(out, t)
}
return out, rows.Err()
}
func (r *Reader) queryColumns(schema, table string) (map[string]*models.Column, error) {
rows, err := r.db.QueryContext(r.ctx, `SELECT COLUMN_NAME, COLUMN_TYPE, IS_NULLABLE, COLUMN_DEFAULT, ORDINAL_POSITION, EXTRA, COLUMN_COMMENT FROM information_schema.COLUMNS WHERE TABLE_SCHEMA = ? AND TABLE_NAME = ? ORDER BY ORDINAL_POSITION`, schema, table)
var onUpdateExtra = regexp.MustCompile(`on update current_timestamp(?:\(\d*\))?`)
func (r *Reader) queryColumns(schema string, table *models.Table) (map[string]*models.Column, error) {
rows, err := r.db.QueryContext(r.ctx, `SELECT COLUMN_NAME, COLUMN_TYPE, IS_NULLABLE, COLUMN_DEFAULT, ORDINAL_POSITION, EXTRA, COLUMN_COMMENT,
CHARACTER_MAXIMUM_LENGTH, NUMERIC_PRECISION, NUMERIC_SCALE, DATETIME_PRECISION, COLLATION_NAME, CHARACTER_SET_NAME, GENERATION_EXPRESSION, SRS_ID
FROM information_schema.COLUMNS WHERE TABLE_SCHEMA=? AND TABLE_NAME=? ORDER BY ORDINAL_POSITION`, schema, table.Name)
if err != nil {
return nil, err
return nil, fmt.Errorf("columns %s.%s: %w", schema, table.Name, err)
}
defer rows.Close()
out := map[string]*models.Column{}
for rows.Next() {
var name, typ, nullable, extra, comment string
var def sql.NullString
var def, collation, charset, generation sql.NullString
var length, precision, scale, datetime, srid sql.NullInt64
var pos int
if err := rows.Scan(&name, &typ, &nullable, &def, &pos, &extra, &comment); err != nil {
if err := rows.Scan(&name, &typ, &nullable, &def, &pos, &extra, &comment, &length, &precision, &scale, &datetime, &collation, &charset, &generation, &srid); err != nil {
return nil, err
}
c := models.InitColumn(name, table, schema)
c.Type = r.mapDataType(typ)
c.NotNull = strings.EqualFold(nullable, "NO")
c.Sequence = uint(pos)
c.Comment = comment
c := models.InitColumn(name, table.Name, schema)
c.Type, c.NotNull, c.Sequence, c.Comment = r.mapDataType(typ), nullable == "NO", uint(pos), comment
// The native declaration preserves mediumint, enum/set, binary/blob sizes,
// unsigned/zerofill and timestamp versus datetime. Canonical dimensions remain editable.
c.Length, c.Precision, c.Scale = int(length.Int64), int(precision.Int64), int(scale.Int64)
if datetime.Valid {
c.Precision = int(datetime.Int64)
}
c.Collation = collation.String
c.Metadata["mysql_type"] = typ
c.Metadata["mysql_type_signature"] = typeSignature(c)
c.Metadata["mysql_charset"] = charset.String
c.Metadata["mysql_datetime_precision"] = datetime.Int64
c.Metadata["mysql_default_expression"] = strings.Contains(extra, "DEFAULT_GENERATED") || strings.HasPrefix(strings.ToLower(def.String), "current_timestamp") && (c.Type == "timestamp")
if def.Valid {
c.Default = def.String
if strings.Contains(extra, "DEFAULT_GENERATED") {
c.Default = unescapeExpression(def.String)
}
if strings.HasPrefix(typ, "bit(") && def.Valid {
c.Metadata["mysql_default_expression"] = true
}
}
c.AutoIncrement = strings.Contains(extra, "auto_increment")
c.Generated, c.GenerationExpression = generation.String != "", unescapeExpression(generation.String)
if strings.Contains(extra, "STORED GENERATED") {
c.Metadata["mysql_generation_storage"] = "STORED"
} else if c.Generated {
c.Metadata["mysql_generation_storage"] = "VIRTUAL"
}
c.Metadata["mysql_invisible"] = strings.Contains(extra, "INVISIBLE")
remaining := onUpdateExtra.ReplaceAllString(strings.ToLower(extra), "")
for _, known := range []string{"auto_increment", "default_generated", "stored generated", "virtual generated", "invisible"} {
remaining = strings.ReplaceAll(remaining, known, "")
}
if remaining = strings.TrimSpace(remaining); remaining != "" {
c.Metadata["mysql_unsupported_extra"] = remaining
}
if i := strings.Index(strings.ToLower(extra), "on update "); i >= 0 {
c.Metadata["mysql_on_update"] = strings.Fields(extra[i+10:])[0]
}
if srid.Valid {
c.Metadata["mysql_srs_id"] = srid.Int64
}
c.AutoIncrement = strings.Contains(strings.ToLower(extra), "auto_increment")
out[name] = c
}
return out, rows.Err()
if err := rows.Err(); err != nil {
return nil, err
}
var tableName, create string
qualified := "`" + strings.ReplaceAll(schema, "`", "``") + "`.`" + strings.ReplaceAll(table.Name, "`", "``") + "`"
if err := r.db.QueryRowContext(r.ctx, "SHOW CREATE TABLE "+qualified).Scan(&tableName, &create); err != nil {
return nil, fmt.Errorf("native defaults %s.%s: %w", schema, table.Name, err)
}
// information_schema AUTO_INCREMENT may be stale because of statistics caching.
for _, c := range out {
if c.AutoIncrement {
table.Metadata["mysql_auto_increment"] = autoIncrementValue(create)
break
}
}
if options := unsupportedNativeOptions(create); len(options) > 0 {
table.Metadata["mysql_unsupported_options"] = strings.Join(options, ", ")
}
for name, sql := range columnDefaults(create) {
c := out[name]
if c == nil || c.Default == nil {
continue
}
if c.Type == "bytea" {
c.Default = sql
}
c.Metadata["mysql_default_sql"] = sql
c.Metadata["mysql_default_value"] = fmt.Sprint(c.Default)
}
return out, nil
}
func typeSignature(c *models.Column) string {
return fmt.Sprintf("%s/%d/%d/%d", c.Type, c.Length, c.Precision, c.Scale)
}
func (r *Reader) queryConstraints(schema, table string) (map[string]*models.Constraint, error) {
rows, err := r.db.QueryContext(r.ctx, `SELECT CONSTRAINT_NAME, CONSTRAINT_TYPE, COLUMN_NAME, REFERENCED_TABLE_SCHEMA, REFERENCED_TABLE_NAME, REFERENCED_COLUMN_NAME, ORDINAL_POSITION FROM information_schema.KEY_COLUMN_USAGE k JOIN information_schema.TABLE_CONSTRAINTS t USING (CONSTRAINT_SCHEMA, TABLE_NAME, CONSTRAINT_NAME) WHERE k.TABLE_SCHEMA=? AND k.TABLE_NAME=? ORDER BY CONSTRAINT_NAME, ORDINAL_POSITION`, schema, table)
rows, err := r.db.QueryContext(r.ctx, `SELECT k.CONSTRAINT_NAME, t.CONSTRAINT_TYPE, k.COLUMN_NAME,
k.REFERENCED_TABLE_SCHEMA, k.REFERENCED_TABLE_NAME, k.REFERENCED_COLUMN_NAME, k.ORDINAL_POSITION,
rc.DELETE_RULE, rc.UPDATE_RULE FROM information_schema.KEY_COLUMN_USAGE k
JOIN information_schema.TABLE_CONSTRAINTS t USING (CONSTRAINT_SCHEMA,TABLE_NAME,CONSTRAINT_NAME)
LEFT JOIN information_schema.REFERENTIAL_CONSTRAINTS rc ON rc.CONSTRAINT_SCHEMA=k.CONSTRAINT_SCHEMA AND rc.TABLE_NAME=k.TABLE_NAME AND rc.CONSTRAINT_NAME=k.CONSTRAINT_NAME
WHERE k.TABLE_SCHEMA=? AND k.TABLE_NAME=? ORDER BY k.CONSTRAINT_NAME,k.ORDINAL_POSITION`, schema, table)
if err != nil {
return nil, err
return nil, fmt.Errorf("constraints %s.%s: %w", schema, table, err)
}
defer rows.Close()
out := map[string]*models.Constraint{}
for rows.Next() {
var name, typ, col, rs, rt, rc string
var name, typ string
var col, rs, rt, rc, del, upd sql.NullString
var pos int
if err := rows.Scan(&name, &typ, &col, &rs, &rt, &rc, &pos); err != nil {
if err := rows.Scan(&name, &typ, &col, &rs, &rt, &rc, &pos, &del, &upd); err != nil {
rows.Close()
return nil, err
}
c := out[name]
@@ -201,31 +313,54 @@ func (r *Reader) queryConstraints(schema, table string) (map[string]*models.Cons
ct = models.ForeignKeyConstraint
}
c = models.InitConstraint(name, ct)
c.Schema = schema
c.Table = table
c.ReferencedSchema = rs
c.ReferencedTable = rt
c.Schema, c.Table = schema, table
c.ReferencedSchema, c.ReferencedTable = rs.String, rt.String
c.OnDelete, c.OnUpdate = del.String, upd.String
out[name] = c
}
c.Columns = append(c.Columns, col)
if rc != "" {
c.ReferencedColumns = append(c.ReferencedColumns, rc)
if col.Valid {
c.Columns = append(c.Columns, col.String)
}
if rc.Valid {
c.ReferencedColumns = append(c.ReferencedColumns, rc.String)
}
}
err = rows.Err()
rows.Close()
if err != nil {
return nil, err
}
rows, err = r.db.QueryContext(r.ctx, `SELECT t.CONSTRAINT_NAME,c.CHECK_CLAUSE,t.ENFORCED FROM information_schema.TABLE_CONSTRAINTS t JOIN information_schema.CHECK_CONSTRAINTS c USING (CONSTRAINT_SCHEMA,CONSTRAINT_NAME) WHERE t.TABLE_SCHEMA=? AND t.TABLE_NAME=? AND t.CONSTRAINT_TYPE='CHECK' ORDER BY t.CONSTRAINT_NAME`, schema, table)
if err != nil {
return nil, fmt.Errorf("checks %s.%s (requires MySQL 8.0.16+): %w", schema, table, err)
}
defer rows.Close()
for rows.Next() {
var name, expr, enforced string
if err := rows.Scan(&name, &expr, &enforced); err != nil {
return nil, err
}
c := models.InitConstraint(name, models.CheckConstraint)
c.Schema, c.Table, c.Expression = schema, table, unescapeExpression(expr)
c.Metadata = map[string]any{"mysql_enforced": enforced == "YES"}
out[name] = c
}
return out, rows.Err()
}
func (r *Reader) queryIndexes(schema, table string) (map[string]*models.Index, error) {
rows, err := r.db.QueryContext(r.ctx, `SELECT INDEX_NAME, NON_UNIQUE, COLUMN_NAME, SEQ_IN_INDEX, INDEX_TYPE FROM information_schema.STATISTICS WHERE TABLE_SCHEMA=? AND TABLE_NAME=? AND INDEX_NAME <> 'PRIMARY' ORDER BY INDEX_NAME, SEQ_IN_INDEX`, schema, table)
rows, err := r.db.QueryContext(r.ctx, `SELECT INDEX_NAME, NON_UNIQUE, COLUMN_NAME, SEQ_IN_INDEX, INDEX_TYPE, EXPRESSION, SUB_PART, COLLATION, IS_VISIBLE, INDEX_COMMENT FROM information_schema.STATISTICS WHERE TABLE_SCHEMA=? AND TABLE_NAME=? ORDER BY INDEX_NAME,SEQ_IN_INDEX`, schema, table)
if err != nil {
return nil, err
return nil, fmt.Errorf("indexes %s.%s (requires MySQL 8.0.13+): %w", schema, table, err)
}
defer rows.Close()
out := map[string]*models.Index{}
for rows.Next() {
var name, col, typ string
var name, typ, visible, comment string
var col, expr, direction sql.NullString
var prefix sql.NullInt64
var non, seq int
if err := rows.Scan(&name, &non, &col, &seq, &typ); err != nil {
if err := rows.Scan(&name, &non, &col, &seq, &typ, &expr, &prefix, &direction, &visible, &comment); err != nil {
return nil, err
}
i := out[name]
@@ -233,22 +368,96 @@ func (r *Reader) queryIndexes(schema, table string) (map[string]*models.Index, e
i = models.InitIndex(name, table, schema)
i.Unique = non == 0
i.Type = strings.ToLower(typ)
i.Comment = comment
i.Metadata["mysql_visible"] = visible == "YES"
out[name] = i
}
i.Columns = append(i.Columns, col)
i.Columns = append(i.Columns, col.String)
i.KeyParts = append(i.KeyParts, models.IndexPart{Name: col.String, Expression: unescapeExpression(expr.String), Length: int(prefix.Int64), Descending: direction.String == "D"})
}
return out, rows.Err()
}
// Record objects outside the supported table-schema contract rather than silently
// discarding them. Writers can warn or reject the conversion before any output.
func (r *Reader) readUnsupported(db *models.Database, schema string) error {
queries := []struct{ kind, sql string }{
{"view", `SELECT TABLE_NAME FROM information_schema.VIEWS WHERE TABLE_SCHEMA=?`},
{"trigger", `SELECT TRIGGER_NAME FROM information_schema.TRIGGERS WHERE TRIGGER_SCHEMA=?`},
{"routine", `SELECT ROUTINE_NAME FROM information_schema.ROUTINES WHERE ROUTINE_SCHEMA=?`},
{"event", `SELECT EVENT_NAME FROM information_schema.EVENTS WHERE EVENT_SCHEMA=?`},
{"partitioned table", `SELECT DISTINCT TABLE_NAME FROM information_schema.PARTITIONS WHERE TABLE_SCHEMA=? AND PARTITION_NAME IS NOT NULL`},
}
var unsupported []string
for _, q := range queries {
rows, err := r.db.QueryContext(r.ctx, q.sql, schema)
if err != nil {
return fmt.Errorf("discover %s: %w", q.kind, err)
}
for rows.Next() {
var name string
if err := rows.Scan(&name); err != nil {
rows.Close()
return err
}
unsupported = append(unsupported, q.kind+" "+schema+"."+name)
}
err = rows.Err()
rows.Close()
if err != nil {
return err
}
}
if len(unsupported) > 0 {
db.Metadata["mysql_unsupported"] = unsupported
for _, s := range db.Schemas {
s.Metadata["mysql_unsupported"] = unsupported
}
if r.options.Progress != nil {
for _, item := range unsupported {
r.options.Progress("warning: unsupported MySQL " + item)
}
}
}
return nil
}
func (r *Reader) deriveRelationship(t *models.Table, c *models.Constraint) {
n := fmt.Sprintf("%s_to_%s", t.Name, c.ReferencedTable)
n := fmt.Sprintf("%s_%s", t.Name, c.Name)
rel := models.InitRelationship(n, models.OneToMany)
rel.FromTable = t.Name
rel.FromSchema = t.Schema
rel.FromTable, rel.FromSchema = t.Name, t.Schema
rel.FromColumns = append([]string(nil), c.Columns...)
rel.ToTable = c.ReferencedTable
rel.ToSchema = c.ReferencedSchema
rel.ToTable, rel.ToSchema = c.ReferencedTable, c.ReferencedSchema
rel.ToColumns = append([]string(nil), c.ReferencedColumns...)
rel.ForeignKey = c.Name
t.Relationships[n] = rel
}
// MySQL information_schema escapes SQL expressions by an additional layer
// (including quote delimiters). This covers defaults, generated columns, checks,
// and functional indexes. Literal defaults are already values and bypass this.
func unescapeExpression(s string) string {
var out strings.Builder
for i := 0; i < len(s); i++ {
if s[i] == '\\' && i+1 < len(s) {
i++
switch s[i] {
case 'n':
out.WriteByte('\n')
case 'r':
out.WriteByte('\r')
case 't':
out.WriteByte('\t')
case '0':
out.WriteByte(0)
case 'Z':
out.WriteByte(26)
default:
out.WriteByte(s[i])
}
} else {
out.WriteByte(s[i])
}
}
return out.String()
}
+16 -1
View File
@@ -3,12 +3,13 @@ package mysql
import (
"testing"
"git.warky.dev/wdevs/relspecgo/pkg/models"
"git.warky.dev/wdevs/relspecgo/pkg/readers"
)
func TestReaderMapDataType(t *testing.T) {
r := NewReader(&readers.ReaderOptions{})
for _, tc := range []struct{ input, want string }{{"varchar(64)", "string"}, {"bigint unsigned", "int64"}, {"datetime", "timestamp"}, {"json", "json"}} {
for _, tc := range []struct{ input, want string }{{"varchar(64)", "string"}, {"bigint unsigned", "uint64"}, {"datetime", "timestamp"}, {"json", "json"}} {
if got := r.mapDataType(tc.input); got != tc.want {
t.Errorf("mapDataType(%q) = %q, want %q", tc.input, got, tc.want)
}
@@ -20,3 +21,17 @@ func TestReaderRequiresConnectionString(t *testing.T) {
t.Fatal("expected missing connection string error")
}
}
func TestRelationshipNamesCannotCollapseDistinctForeignKeys(t *testing.T) {
table := models.InitTable("t", "app")
reader := NewReader(&readers.ReaderOptions{})
for _, c := range []*models.Constraint{
{Name: "a_to_b", ReferencedTable: "c"},
{Name: "a", ReferencedTable: "b_to_c"},
} {
reader.deriveRelationship(table, c)
}
if len(table.Relationships) != 2 {
t.Fatalf("distinct foreign keys collapsed: %+v", table.Relationships)
}
}
+145
View File
@@ -0,0 +1,145 @@
package mysql
import (
"os"
"path/filepath"
"strings"
"testing"
"git.warky.dev/wdevs/relspecgo/pkg/models"
"git.warky.dev/wdevs/relspecgo/pkg/transform"
"git.warky.dev/wdevs/relspecgo/pkg/writers"
)
func TestStrictConversionPreflightsAllObjects(t *testing.T) {
db := shopDB()
unsupported := models.InitTable("unsupported", "shop")
col := models.InitColumn("opaque", "unsupported", "shop")
col.Type = "unknown_type"
unsupported.Columns[col.Name] = col
index := models.InitIndex("partial", "unsupported", "shop")
index.Columns = []string{col.Name}
index.Where = "opaque IS NOT NULL"
index.Include = []string{"other"}
unsupported.Indexes[index.Name] = index
db.Schemas[0].Tables = append(db.Schemas[0].Tables, unsupported)
out := filepath.Join(t.TempDir(), "existing.sql")
if err := os.WriteFile(out, []byte("keep existing contents"), 0600); err != nil {
t.Fatal(err)
}
opts := &writers.WriterOptions{StrictConversion: true, OutputPath: out}
err := NewWriter(opts).WriteDatabase(db)
for _, want := range []string{"lossy MySQL conversion rejected", "unknown type", "partial predicate", "INCLUDE"} {
if err == nil || !strings.Contains(err.Error(), want) {
t.Fatalf("expected %s: %v", want, err)
}
}
bytes, err := os.ReadFile(out)
if err != nil || string(bytes) != "keep existing contents" {
t.Fatalf("destination changed: %q %v", bytes, err)
}
opts.Metadata = map[string]any{"connection_string": "invalid DSN"}
if err := NewWriter(opts).WriteDatabase(db); err == nil || !strings.Contains(err.Error(), "lossy MySQL") {
t.Fatalf("connected before strict validation: %v", err)
}
opts.StrictConversion = false
opts.Metadata = nil
var warnings []string
opts.Warning = func(s string) { warnings = append(warnings, s) }
if err := NewWriter(opts).WriteDatabase(db); err != nil {
t.Fatal(err)
}
if len(warnings) != 3 {
t.Fatalf("warnings: %v", warnings)
}
}
func TestUnsupportedModelFeaturesDiagnosed(t *testing.T) {
db := shopDB()
s := db.Schemas[0]
table := s.Tables[0]
s.Metadata["mysql_unsupported"] = []any{"trigger shop.audit", "partitioned table shop.archive"}
c := table.Columns["id"]
c.Type = "timestamptz"
c.Identity = true
c.Metadata["mysql_srs_id"] = 4326
check := models.InitConstraint("deferred", models.CheckConstraint)
check.Expression = "id > 0"
yes := true
check.Deferrable = &yes
table.Constraints[check.Name] = check
i := models.InitIndex("gin", "users", "shop")
i.Type = "gin"
i.Columns = []string{"id"}
table.Indexes[i.Name] = i
_, err := NewWriter(&writers.WriterOptions{StrictConversion: true}).Statements(db)
for _, want := range []string{"trigger shop.audit", "partitioned table", "time zone", "identity", "mysql_srs_id", "deferrable", "unsupported index type gin"} {
if err == nil || !strings.Contains(err.Error(), want) {
t.Errorf("missing diagnostic %q: %v", want, err)
}
}
}
func TestCanonicalDimensionsAndNativeEdit(t *testing.T) {
db := shopDB()
c := db.Schemas[0].Tables[0].Columns["age"]
c.Type = "decimal"
c.Precision = 12
c.Scale = 4
if got := columnType(c); got != "DECIMAL(12,4)" {
t.Fatal(got)
}
c.Type = "uint64"
c.Precision = 0
c.Scale = 0
if got := columnType(c); got != "BIGINT UNSIGNED" {
t.Fatal(got)
}
c.Metadata["mysql_type"] = "bigint unsigned"
c.Metadata["mysql_type_signature"] = signature(c)
c.Type = "int64"
_, err := NewWriter(&writers.WriterOptions{StrictConversion: true}).Statements(db)
if err == nil || !strings.Contains(err.Error(), "override native declaration") {
t.Fatalf("native metadata silently overrode edit: %v", err)
}
if got := columnType(c); got != "BIGINT" {
t.Fatal(got)
}
}
func TestLowercaseIndexPartsPreserveSQL(t *testing.T) {
db := shopDB()
table := db.Schemas[0].Tables[0]
c := models.InitColumn("Mixed", "users", "shop")
c.Type = "string"
table.Columns[c.Name] = c
i := models.InitIndex("MixedIndex", "users", "shop")
i.KeyParts = []models.IndexPart{{Name: "Mixed", Length: 8, Descending: true}, {Expression: "lower(`Mixed`)"}}
table.Indexes[i.Name] = i
model, err := transform.LowercaseIdentifiers(db)
if err != nil {
t.Fatal(err)
}
parts := model.Schemas[0].Tables[0].Indexes["mixedindex"].KeyParts
if parts[0].Name != "mixed" || parts[1].Expression != "lower(`Mixed`)" || i.KeyParts[0].Name != "Mixed" {
t.Fatalf("lowercase parts: %+v", parts)
}
}
func TestLowercasePrimaryIndexIsNotDuplicated(t *testing.T) {
db := shopDB()
table := db.Schemas[0].Tables[0]
i := models.InitIndex("PRIMARY", table.Name, table.Schema)
i.Unique = true
i.Type = "btree"
i.Columns = []string{"id"}
i.KeyParts = []models.IndexPart{{Name: "id", Descending: true}}
table.Indexes[i.Name] = i
statements, err := NewWriter(&writers.WriterOptions{StrictConversion: true, LowercaseIdentifiers: true}).Statements(db)
if err != nil {
t.Fatal(err)
}
if !strings.Contains(statements[0], "PRIMARY KEY (`id` DESC)") || strings.Contains(statements[0], "KEY `primary`") {
t.Fatalf("primary key duplicated or attributes lost: %s", statements[0])
}
}
+184
View File
@@ -0,0 +1,184 @@
package mysql
import (
"fmt"
"sort"
"strings"
"git.warky.dev/wdevs/relspecgo/pkg/mariadb"
"git.warky.dev/wdevs/relspecgo/pkg/models"
)
// Diagnostics cover modeled schema semantics and features discovered by the
// MySQL reader. They do not certify arbitrary SQL expressions or catalog grants.
func conversionDiagnostics(db *models.Database) []string {
var out []string
warn := func(path, feature string) { out = append(out, path+": "+feature) }
unsupported := func(m map[string]any) {
switch values := m["mysql_unsupported"].(type) {
case []string:
for _, v := range values {
warn(v, "unsupported object")
}
case []any:
for _, v := range values {
warn(fmt.Sprint(v), "unsupported object")
}
}
}
metadata := func(path string, m map[string]any, allowed string) {
if len(models.GetDirectives(m)) > 0 {
warn(path, "dialect directives are not emitted")
}
for _, key := range keys(m) {
if strings.HasPrefix(key, "mysql_") && !strings.Contains(" "+allowed+" ", " "+key+" ") {
warn(path, "unsupported metadata "+key+": "+fmt.Sprint(m[key]))
}
}
}
metadata("database", db.Metadata, "mysql_unsupported mysql_sql_mode")
modeWarning := func(path string, m map[string]any) {
if mode := unsupportedMode(metaString(m, "mysql_sql_mode")); mode != "" {
warn(path, "unsupported source SQL mode "+mode+"; generated SQL requires standard quotes and backslash escaping")
}
}
modeWarning("database", db.Metadata)
unsupported(db.Metadata)
if len(db.Domains) > 0 {
warn("database", "domains are not emitted")
}
for _, s := range db.Schemas {
path := s.Name
metadata(path, s.Metadata, "mysql_unsupported mysql_sql_mode")
modeWarning(path, s.Metadata)
unsupported(s.Metadata)
if len(s.Views) > 0 {
warn(path, "views are not emitted")
}
if len(s.Sequences) > 0 {
warn(path, "standalone sequences are not emitted")
}
if len(s.Enums) > 0 {
warn(path, "standalone enum types are not emitted; use native column enum types")
}
if len(s.Scripts) > 0 {
warn(path, "scripts are not emitted")
}
if s.Owner != "" || len(s.Permissions) > 0 {
warn(path, "ownership and permissions are not emitted")
}
for _, t := range s.Tables {
tp := path + "." + t.Name
modeWarning(tp, t.Metadata)
if t.Tablespace != "" {
warn(tp, "tablespace is not emitted")
}
metadata(tp, t.Metadata, "mysql_engine mysql_collation mysql_create_options mysql_auto_increment mysql_sql_mode")
if options := metaString(t.Metadata, "mysql_create_options"); options != "" {
warn(tp, "table CREATE_OPTIONS are not emitted: "+options)
}
for _, name := range keys(t.Columns) {
c := t.Columns[name]
cp := tp + "." + c.Name
metadata(cp, c.Metadata, "mysql_type mysql_type_signature mysql_charset mysql_datetime_precision mysql_default_expression mysql_default_sql mysql_default_value mysql_generation_storage mysql_invisible mysql_on_update")
if _, known := mariadb.CanonicalToMariaDBTypes[strings.ToLower(c.Type)]; !known && metaString(c.Metadata, "mysql_type") == "" {
warn(cp, "unknown type "+c.Type+" maps to "+columnType(c))
}
if native := metaString(c.Metadata, "mysql_type"); native != "" && metaString(c.Metadata, "mysql_type_signature") != signature(c) {
warn(cp, "edited canonical type/dimensions override native declaration "+native)
}
if c.Type == "timestamptz" {
warn(cp, "time zone semantics are not supported")
}
if c.Type == "jsonb" {
warn(cp, "jsonb storage semantics become JSON")
}
if c.Identity {
warn(cp, "identity semantics are not emitted; use auto_increment")
}
if c.Generated && c.GenerationExpression == "" {
warn(cp, "generated column has no expression")
}
if !c.Generated && c.GenerationExpression != "" {
warn(cp, "generation expression requires generated=true")
}
if c.Generated && (c.Default != nil || c.AutoIncrement) {
warn(cp, "generated column cannot preserve a default or auto_increment")
}
if storage := metaString(c.Metadata, "mysql_generation_storage"); storage != "" && storage != "STORED" && storage != "VIRTUAL" {
warn(cp, "unsupported generation storage "+storage)
}
}
for _, name := range keys(t.Constraints) {
c := t.Constraints[name]
cp := tp + " constraint " + c.Name
metadata(cp, c.Metadata, "mysql_enforced")
if c.Deferrable != nil && *c.Deferrable || c.InitiallyDeferred != nil && *c.InitiallyDeferred {
warn(cp, "deferrable constraints are not supported")
}
switch c.Type {
case models.PrimaryKeyConstraint, models.UniqueConstraint:
case models.CheckConstraint:
if c.Expression == "" {
warn(cp, "check constraint has no expression")
}
case models.ForeignKeyConstraint:
engine := metaString(t.Metadata, "mysql_engine")
if engine != "" && !strings.EqualFold(engine, "InnoDB") && !strings.EqualFold(engine, "NDB") {
warn(cp, "foreign keys are not supported by engine "+engine)
}
for _, action := range []string{c.OnDelete, c.OnUpdate} {
switch strings.ToUpper(action) {
case "", "CASCADE", "SET NULL", "RESTRICT", "NO ACTION":
default:
warn(cp, "unsupported referential action is not emitted: "+action)
}
}
default:
warn(cp, "unsupported constraint type "+string(c.Type))
}
}
for _, name := range keys(t.Indexes) {
i := t.Indexes[name]
ip := tp + " index " + i.Name
metadata(ip, i.Metadata, "mysql_visible")
if i.Where != "" {
warn(ip, "partial predicate is not emitted")
}
if len(i.Include) > 0 {
warn(ip, "INCLUDE columns are not emitted")
}
if i.Concurrent {
warn(ip, "concurrent creation is not supported")
}
if strings.EqualFold(i.Type, "hash") && !strings.EqualFold(metaString(t.Metadata, "mysql_engine"), "MEMORY") {
warn(ip, "HASH index requires a compatible engine; InnoDB converts it to BTREE")
}
switch strings.ToUpper(i.Type) {
case "", "BTREE", "HASH", "FULLTEXT", "SPATIAL":
default:
warn(ip, "unsupported index type "+i.Type+"; index is not emitted")
}
for _, p := range i.KeyParts {
if (p.Name == "") == (p.Expression == "") {
warn(ip, "key part requires exactly one name or expression")
}
}
}
for _, name := range keys(t.Relationships) {
r := t.Relationships[name]
if t.Constraints[r.ForeignKey] == nil {
warn(tp+" relationship "+r.Name, "no backing foreign key constraint; relationship is not emitted")
}
}
}
}
sort.Strings(out)
unique := out[:0]
for _, d := range out {
if len(unique) == 0 || unique[len(unique)-1] != d {
unique = append(unique, d)
}
}
return unique
}
+140
View File
@@ -0,0 +1,140 @@
package mysql_test
import (
"database/sql"
"fmt"
"git.warky.dev/wdevs/relspecgo/pkg/models"
driver "github.com/go-sql-driver/mysql"
"os"
"path/filepath"
"strings"
"testing"
"time"
"git.warky.dev/wdevs/relspecgo/pkg/readers"
rmysql "git.warky.dev/wdevs/relspecgo/pkg/readers/mysql"
"git.warky.dev/wdevs/relspecgo/pkg/writers"
wmysql "git.warky.dev/wdevs/relspecgo/pkg/writers/mysql"
)
func TestLiveUnsupportedObjectsDiagnosed(t *testing.T) {
conn := os.Getenv("RELSPEC_TEST_MYSQL_CONN")
if conn == "" {
t.Skip("RELSPEC_TEST_MYSQL_CONN is not set")
}
db, err := sql.Open("mysql", conn)
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { db.Close() })
prefix := fmt.Sprintf("unsupported_%d", time.Now().UnixNano())
table, view, trigger := prefix+"_table", prefix+"_view", prefix+"_trigger"
t.Cleanup(func() {
for _, stmt := range []string{"DROP VIEW IF EXISTS " + view, "DROP TRIGGER IF EXISTS " + trigger, "DROP TABLE IF EXISTS " + table} {
if _, err := db.Exec(stmt); err != nil {
t.Error(err)
}
}
})
for _, stmt := range []string{
"CREATE TABLE " + table + " (id INT PRIMARY KEY) PARTITION BY HASH(id) PARTITIONS 2",
"CREATE VIEW " + view + " AS SELECT id FROM " + table,
"CREATE TRIGGER " + trigger + " BEFORE INSERT ON " + table + " FOR EACH ROW SET NEW.id = NEW.id + 1",
} {
if _, err := db.Exec(stmt); err != nil {
t.Fatalf("%s: %v", stmt, err)
}
}
model, err := rmysql.NewReader(&readers.ReaderOptions{ConnectionString: conn}).ReadDatabase()
if err != nil {
t.Fatal(err)
}
path := filepath.Join(t.TempDir(), "schema.sql")
if err := os.WriteFile(path, []byte("original"), 0600); err != nil {
t.Fatal(err)
}
err = wmysql.NewWriter(&writers.WriterOptions{StrictConversion: true, OutputPath: path}).WriteDatabase(model)
for _, want := range []string{"partitioned table", view, trigger} {
if err == nil || !strings.Contains(err.Error(), want) {
t.Errorf("missing diagnostic %s: %v", want, err)
}
}
bytes, readErr := os.ReadFile(path)
if readErr != nil || string(bytes) != "original" {
t.Fatalf("strict failure touched file: %s %v", bytes, readErr)
}
// Schema-filtered/library writes retain discovery diagnostics as well.
err = wmysql.NewWriter(&writers.WriterOptions{StrictConversion: true}).WriteSchema(model.Schemas[0])
if err == nil || !strings.Contains(err.Error(), trigger) {
t.Fatalf("filtered schema lost diagnostics: %v", err)
}
var warnings []string
_, err = wmysql.NewWriter(&writers.WriterOptions{Warning: func(s string) { warnings = append(warnings, s) }}).Statements(model)
if err != nil {
t.Fatal(err)
}
for _, want := range []string{view, trigger, table} {
found := false
for _, warning := range warnings {
if strings.Contains(warning, want) {
found = true
}
}
if !found {
t.Errorf("no warning for %s: %v", want, warnings)
}
}
}
func TestLiveUnsupportedSQLModeFailsBeforeDDL(t *testing.T) {
conn := os.Getenv("RELSPEC_TEST_MYSQL_CONN")
if conn == "" {
t.Skip("RELSPEC_TEST_MYSQL_CONN is not set")
}
for _, mode := range []string{"NO_BACKSLASH_ESCAPES", "ANSI_QUOTES"} {
t.Run(mode, func(t *testing.T) {
cfg, err := driver.ParseDSN(conn)
if err != nil {
t.Fatal(err)
}
if cfg.Params == nil {
cfg.Params = map[string]string{}
}
cfg.Params["sql_mode"] = "'" + mode + "'"
modeConn := cfg.FormatDSN()
model, err := rmysql.NewReader(&readers.ReaderOptions{ConnectionString: modeConn}).ReadDatabase()
if err != nil {
t.Fatal(err)
}
if _, err := wmysql.NewWriter(&writers.WriterOptions{StrictConversion: true}).Statements(model); err == nil || !strings.Contains(err.Error(), mode) {
t.Fatalf("source mode not diagnosed: %v", err)
}
db, err := sql.Open("mysql", conn)
if err != nil {
t.Fatal(err)
}
defer db.Close()
name := fmt.Sprintf("mode_%d", time.Now().UnixNano())
defer db.Exec("DROP TABLE IF EXISTS " + name)
target := models.InitDatabase(cfg.DBName)
s := models.InitSchema(cfg.DBName)
target.Schemas = append(target.Schemas, s)
table := models.InitTable(name, s.Name)
s.Tables = append(s.Tables, table)
c := models.InitColumn("id", name, s.Name)
c.Type = "int"
table.Columns[c.Name] = c
err = wmysql.NewWriter(&writers.WriterOptions{StrictConversion: true, Metadata: map[string]any{"connection_string": modeConn}}).WriteDatabase(target)
if err == nil || !strings.Contains(err.Error(), mode) {
t.Fatalf("target mode not rejected: %v", err)
}
var count int
if err := db.QueryRow("SELECT COUNT(*) FROM information_schema.TABLES WHERE TABLE_SCHEMA=? AND TABLE_NAME=?", cfg.DBName, name).Scan(&count); err != nil {
t.Fatal(err)
}
if count != 0 {
t.Fatal("mode rejection occurred after DDL")
}
})
}
}
+242
View File
@@ -0,0 +1,242 @@
package mysql_test
import (
"database/sql"
"encoding/json"
"fmt"
"gopkg.in/yaml.v3"
"os"
"reflect"
"sort"
"strings"
"testing"
"time"
"git.warky.dev/wdevs/relspecgo/pkg/models"
"git.warky.dev/wdevs/relspecgo/pkg/readers"
rmysql "git.warky.dev/wdevs/relspecgo/pkg/readers/mysql"
"git.warky.dev/wdevs/relspecgo/pkg/writers"
wmysql "git.warky.dev/wdevs/relspecgo/pkg/writers/mysql"
driver "github.com/go-sql-driver/mysql"
)
func TestLiveSchemaRoundTrip(t *testing.T) {
conn := os.Getenv("RELSPEC_TEST_MYSQL_CONN")
if conn == "" {
t.Skip("RELSPEC_TEST_MYSQL_CONN is not set")
}
db, err := sql.Open("mysql", conn)
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { db.Close() })
prefix := fmt.Sprintf("rt_%d", time.Now().UnixNano())
parent, child := prefix+"_zparent", prefix+"_achild"
q := func(n string) string { return "`" + strings.ReplaceAll(n, "`", "``") + "`" }
exec := func(stmt string) {
t.Helper()
if _, err := db.Exec(stmt); err != nil {
t.Fatalf("%v\n%s", err, stmt)
}
}
t.Cleanup(func() {
for _, name := range []string{child + "_copy", child, parent + "_copy", parent} {
if _, err := db.Exec("DROP TABLE IF EXISTS " + q(name)); err != nil {
t.Errorf("cleanup %s: %v", name, err)
}
}
})
exec("CREATE TABLE " + q(parent) + ` (
id BIGINT UNSIGNED NOT NULL AUTO_INCREMENT,
code CHAR(3) NOT NULL,
email VARCHAR(100) NOT NULL,
label VARCHAR(64) DEFAULT 'don''t (change)' COMMENT 'column''s comment',
amount DECIMAL(12,4) NOT NULL DEFAULT 3.2500,
medium MEDIUMINT UNSIGNED,
flags BIT(8) DEFAULT b'00001010',
status ENUM('one','two','don''t, split') DEFAULT 'don''t, split',
choices SET('a','b') DEFAULT 'a,b',
stamp TIMESTAMP(6) NULL DEFAULT CURRENT_TIMESTAMP(6) ON UPDATE CURRENT_TIMESTAMP(6),
duration TIME(4),
bytes VARBINARY(16),
binary_def VARBINARY(8) DEFAULT X'00275CFF',
slash VARCHAR(64) DEFAULT 'C:\\temp\\file',
uuid_value CHAR(36) DEFAULT (uuid()),
hidden_col INT INVISIBLE,
dt DATETIME(6) DEFAULT '2020-01-02 03:04:05.123456',
payload MEDIUMBLOB,
doc TEXT DEFAULT ('first;
-- data, not a comment
last'),
generated_amount DECIMAL(13,4) GENERATED ALWAYS AS (amount * 2) STORED,
virtual_label VARCHAR(80) GENERATED ALWAYS AS (concat('prefix:',lower(label))) VIRTUAL,
PRIMARY KEY (id,code),
UNIQUE KEY email_unique (email),
UNIQUE KEY pair_unique (label,email),
UNIQUE KEY unique_expression ((concat(lower(code),'-x')),email),
KEY prefix_desc (label(10) DESC),
KEY expression_key ((lower(email))),
KEY invisible_key (medium) INVISIBLE COMMENT 'index''s comment',
FULLTEXT KEY text_search (doc),
CONSTRAINT ` + q(prefix+"_amount_check") + ` CHECK (amount >= 0),
CONSTRAINT ` + q(prefix+"_optional_check") + ` CHECK (medium > 0 AND status <> 'forbidden;-- value') NOT ENFORCED
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci COMMENT='table''s comment'`)
exec("CREATE TABLE " + q(child) + ` (
id INT PRIMARY KEY,
parent_id BIGINT UNSIGNED,
parent_code CHAR(3),
parent_email VARCHAR(100),
CONSTRAINT ` + q(prefix+"_composite_fk") + ` FOREIGN KEY (parent_id,parent_code) REFERENCES ` + q(parent) + ` (id,code) ON DELETE SET NULL ON UPDATE CASCADE,
CONSTRAINT ` + q(prefix+"_email_fk") + ` FOREIGN KEY (parent_email) REFERENCES ` + q(parent) + ` (email) ON DELETE CASCADE ON UPDATE RESTRICT
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci`)
readerConfig, err := driver.ParseDSN(conn)
if err != nil {
t.Fatal(err)
}
if readerConfig.Params == nil {
readerConfig.Params = map[string]string{}
}
readerConfig.Params["sql_quote_show_create"] = "0" // reader must ensure quoted native metadata
read := func() *models.Database {
t.Helper()
model, err := rmysql.NewReader(&readers.ReaderOptions{ConnectionString: readerConfig.FormatDSN()}).ReadDatabase()
if err != nil {
t.Fatal(err)
}
return model
}
model := read()
var selected []*models.Table
for _, table := range model.Schemas[0].Tables {
if table.Name == parent || table.Name == child {
selected = append(selected, table)
}
}
if len(selected) != 2 {
t.Fatalf("read %d fixture tables", len(selected))
}
model.Schemas[0].Tables = selected
model.Metadata = nil
model.Schemas[0].Metadata = nil // unrelated pre-existing test objects are outside this fixture
for _, table := range selected {
if table.Name == parent {
if table.Columns["id"].Type != "uint64" {
t.Fatal("unsigned type lost")
}
if table.Columns["amount"].Precision != 12 || table.Columns["amount"].Scale != 4 {
t.Fatal("decimal dimensions lost")
}
if table.Indexes["expression_key"].KeyParts[0].Expression == "" {
t.Fatal("functional index expression lost")
}
} else if len(table.Relationships) != 2 {
t.Fatalf("foreign key relationships collapsed: %d", len(table.Relationships))
}
}
// Exercise the serialized model boundary as well as the native reader objects.
encoded, err := json.Marshal(model)
if err != nil {
t.Fatal(err)
}
var target models.Database
if err := json.Unmarshal(encoded, &target); err != nil {
t.Fatal(err)
}
yamlModel, err := yaml.Marshal(&target)
if err != nil {
t.Fatal(err)
}
target = models.Database{}
if err := yaml.Unmarshal(yamlModel, &target); err != nil {
t.Fatal(err)
}
for _, table := range target.Schemas[0].Tables {
table.Name += "_copy"
for _, col := range table.Columns {
col.Table = table.Name
}
for _, index := range table.Indexes {
index.Table = table.Name
}
constraints := map[string]*models.Constraint{}
for _, c := range table.Constraints {
c.Table = table.Name
if c.Type == models.ForeignKeyConstraint {
c.ReferencedTable += "_copy"
}
if c.Type == models.ForeignKeyConstraint || c.Type == models.CheckConstraint {
c.Name += "_copy"
}
constraints[c.Name] = c
}
table.Constraints = constraints
for _, r := range table.Relationships {
r.FromTable += "_copy"
r.ToTable += "_copy"
r.ForeignKey += "_copy"
}
}
opts := &writers.WriterOptions{StrictConversion: true, Metadata: map[string]any{"connection_string": conn}}
if err := wmysql.NewWriter(opts).WriteDatabase(&target); err != nil {
t.Fatal(err)
}
show := func(table string) string {
t.Helper()
var name, ddl string
if err := db.QueryRow("SHOW CREATE TABLE "+q(table)).Scan(&name, &ddl); err != nil {
t.Fatal(err)
}
ddl = strings.ReplaceAll(ddl, "_copy", "")
// Index creation order is not semantic. Keep column and constraint order intact.
lines := strings.Split(ddl, "\n")
var indexes []string
for _, line := range lines {
if strings.HasPrefix(strings.TrimSpace(line), "KEY ") {
indexes = append(indexes, line)
}
}
sort.Strings(indexes)
n := 0
for i, line := range lines {
if strings.HasPrefix(strings.TrimSpace(line), "KEY ") {
lines[i] = indexes[n]
n++
}
}
return strings.Join(lines, "\n")
}
for _, name := range []string{parent, child} {
a, b := show(name), show(name+"_copy")
if a != b {
t.Errorf("schema mismatch\nsource:\n%s\ntarget:\n%s", a, b)
}
}
again := read()
for _, table := range again.Schemas[0].Tables {
if table.Name == parent+"_copy" && !reflect.DeepEqual(table.Columns["amount"].Metadata, selected[1].Columns["amount"].Metadata) {
t.Error("type metadata changed on second read")
}
}
// Verify constraints and defaults as behavior, after comparing untouched schemas.
exec("INSERT INTO " + q(parent+"_copy") + " (id,code,email) VALUES (18446744073709551614,'abc','one@example.com')")
exec("INSERT INTO " + q(child+"_copy") + " (id,parent_id,parent_code,parent_email) VALUES (1,18446744073709551614,'abc','one@example.com')")
if _, err := db.Exec("INSERT INTO " + q(child+"_copy") + " (id,parent_id,parent_code) VALUES (2,123,'abc')"); err == nil {
t.Error("foreign key is not enforced")
}
if _, err := db.Exec("UPDATE " + q(parent+"_copy") + " SET amount=-1"); err == nil {
t.Error("check is not enforced")
}
var label, doc, binaryHex, slash string
var flags int
if err := db.QueryRow("SELECT label,doc,flags+0,HEX(binary_def),slash FROM "+q(parent+"_copy")).Scan(&label, &doc, &flags, &binaryHex, &slash); err != nil {
t.Fatal(err)
}
for _, table := range read().Schemas[0].Tables {
if table.Name == parent+"_copy" && table.Metadata["mysql_auto_increment"] != "18446744073709551615" {
t.Errorf("large auto-increment counter lost: %v", table.Metadata["mysql_auto_increment"])
}
}
if label != "don't (change)" || doc != "first;\n-- data, not a comment\nlast" || flags != 10 || binaryHex != "00275CFF" || slash != `C:\temp\file` {
t.Errorf("defaults corrupted: %q %q %d %s %q", label, doc, flags, binaryHex, slash)
}
}
+339 -96
View File
@@ -7,13 +7,14 @@ import (
"io"
"os"
"sort"
"strconv"
"strings"
_ "github.com/go-sql-driver/mysql"
"git.warky.dev/wdevs/relspecgo/pkg/mariadb"
"git.warky.dev/wdevs/relspecgo/pkg/models"
"git.warky.dev/wdevs/relspecgo/pkg/transform"
"git.warky.dev/wdevs/relspecgo/pkg/writers"
_ "github.com/go-sql-driver/mysql"
)
type Writer struct {
@@ -22,88 +23,192 @@ type Writer struct {
}
func NewWriter(options *writers.WriterOptions) *Writer { return &Writer{options: options} }
func (w *Writer) WriteDatabase(db *models.Database) error {
// Statements preflights the whole model, emits conversion diagnostics, and returns
// executable units. No output files or database connections are opened here.
func (w *Writer) Statements(db *models.Database) ([]string, error) {
if w.options == nil {
return fmt.Errorf("writer options are required")
return nil, fmt.Errorf("writer options are required")
}
if conn, ok := w.options.Metadata["connection_string"].(string); ok && conn != "" {
return w.execute(db, conn)
if db == nil {
return nil, fmt.Errorf("database is required")
}
release, err := w.openOutput()
if err != nil {
return err
}
defer release()
return w.writeDatabaseDDL(db)
}
func (w *Writer) writeDatabaseDDL(db *models.Database) error {
if _, err := fmt.Fprintf(w.writer, "-- MySQL Database Schema\n-- Database: %s\n-- Generated by RelSpec\n\n", db.Name); err != nil {
return err
}
for _, s := range db.Schemas {
if err := w.WriteSchema(s); err != nil {
return err
}
}
return nil
}
// openOutput points w.writer at the configured destination (output file or
// stdout) when none is set, and returns a func that releases it again.
func (w *Writer) openOutput() (func(), error) {
if w.writer != nil {
return func() {}, nil
}
if w.options != nil && w.options.OutputPath != "" {
f, err := os.Create(w.options.OutputPath)
if w.options.LowercaseIdentifiers {
var err error
db, err = transform.LowercaseIdentifiers(db)
if err != nil {
return nil, err
}
w.writer = f
return func() {
f.Close()
w.writer = nil
}, nil
}
w.writer = os.Stdout
return func() { w.writer = nil }, nil
diagnostics := conversionDiagnostics(db)
if len(diagnostics) > 0 && w.options.StrictConversion {
return nil, fmt.Errorf("lossy MySQL conversion rejected:\n- %s", strings.Join(diagnostics, "\n- "))
}
for _, d := range diagnostics {
if w.options.Warning != nil {
w.options.Warning(d)
} else {
fmt.Fprintln(os.Stderr, "warning: MySQL conversion: "+d)
}
}
return databaseStatements(db), nil
}
func (w *Writer) WriteSchema(s *models.Schema) error {
release, err := w.openOutput()
func (w *Writer) WriteDatabase(db *models.Database) error {
statements, err := w.Statements(db)
if err != nil {
return err
}
defer release()
for _, t := range s.Tables {
if err := w.writeTable(s, t); err != nil {
if conn, ok := w.options.Metadata["connection_string"].(string); ok && conn != "" {
return w.execute(statements, conn)
}
if w.writer != nil {
return writeStatements(w.writer, statements, db.Name)
}
if w.options.OutputPath == "" {
return writeStatements(os.Stdout, statements, db.Name)
}
f, err := os.Create(w.options.OutputPath)
if err != nil {
return err
}
err = writeStatements(f, statements, db.Name)
closeErr := f.Close()
if err != nil {
return err
}
return closeErr
}
func writeStatements(out io.Writer, statements []string, name string) error {
name = strings.NewReplacer("\n", " ", "\r", " ").Replace(name)
if _, err := fmt.Fprintf(out, "-- Database: %s\n", name); err != nil {
return err
}
if _, err := io.WriteString(out, "-- MySQL table schema; generated by RelSpec\n\n"); err != nil {
return err
}
for _, stmt := range statements {
if _, err := io.WriteString(out, stmt); err != nil {
return err
}
}
return nil
}
func (w *Writer) WriteSchema(s *models.Schema) error {
return w.WriteDatabase(&models.Database{Schemas: []*models.Schema{s}})
}
func (w *Writer) WriteTable(t *models.Table) error {
release, err := w.openOutput()
if err != nil {
return err
return w.WriteSchema(&models.Schema{Name: "", Tables: []*models.Table{t}})
}
func qualified(schema, name string) string {
if schema == "" {
return quote(name)
}
defer release()
return w.writeTable(nil, t)
return quote(schema) + "." + quote(name)
}
func keys[T any](m map[string]T) []string {
out := make([]string, 0, len(m))
for k := range m {
out = append(out, k)
}
sort.Strings(out)
return out
}
func metaString(m map[string]any, key string) string { v, _ := m[key].(string); return v }
func metaBool(m map[string]any, key string, def bool) bool {
v, ok := m[key].(bool)
if ok {
return v
}
return def
}
func signature(c *models.Column) string {
return fmt.Sprintf("%s/%d/%d/%d", c.Type, c.Length, c.Precision, c.Scale)
}
func columnType(c *models.Column) string {
if native := metaString(c.Metadata, "mysql_type"); native != "" && metaString(c.Metadata, "mysql_type_signature") == signature(c) {
return native
}
typ := mariadb.ConvertCanonicalToMariaDB(c.Type)
switch strings.ToLower(c.Type) {
case "string":
if c.Length > 0 {
typ = fmt.Sprintf("VARCHAR(%d)", c.Length)
}
case "decimal":
if c.Precision > 0 {
typ = fmt.Sprintf("DECIMAL(%d,%d)", c.Precision, c.Scale)
}
case "timestamp", "time":
if c.Precision > 0 {
typ += fmt.Sprintf("(%d)", c.Precision)
}
}
return typ
}
func (w *Writer) writeTable(s *models.Schema, t *models.Table) error {
_, err := io.WriteString(w.writer, tableStatement(s, t))
return err
// MySQL string literals use doubled quotes and escaped backslashes. The output
// assumes the server's standard SQL mode (NO_BACKSLASH_ESCAPES is not enabled).
func literal(s string) string {
return "'" + strings.ReplaceAll(strings.ReplaceAll(s, "\\", "\\\\"), "'", "''") + "'"
}
func defaultSQL(c *models.Column) string {
value := fmt.Sprint(c.Default)
if native := metaString(c.Metadata, "mysql_default_sql"); native != "" && metaString(c.Metadata, "mysql_default_value") == value {
return native
}
if strings.HasPrefix(metaString(c.Metadata, "mysql_type"), "bit(") && strings.HasPrefix(value, "b'") && strings.HasSuffix(value, "'") && strings.Trim(value[2:len(value)-1], "01") == "" {
return value
}
if metaBool(c.Metadata, "mysql_default_expression", false) {
if strings.HasPrefix(strings.ToUpper(value), "CURRENT_TIMESTAMP") {
return value
}
return "(" + value + ")"
}
switch strings.ToLower(c.Type) {
case "text", "json", "jsonb", "bytea":
return "(" + literal(value) + ")"
case "string", "uuid", "date", "time", "timestamp", "timestamptz":
return literal(value)
case "bool", "boolean":
if value == "true" {
return "1"
}
if value == "false" {
return "0"
}
}
// Numeric defaults are values, never executable fragments.
if _, err := strconv.ParseFloat(value, 64); err == nil {
return value
}
return literal(value)
}
func indexParts(i *models.Index) string {
if len(i.KeyParts) == 0 {
return quoted(i.Columns)
}
parts := make([]string, 0, len(i.KeyParts))
for _, p := range i.KeyParts {
s := quote(p.Name)
if p.Expression != "" {
s = "(" + p.Expression + ")"
}
if p.Length > 0 {
s += fmt.Sprintf("(%d)", p.Length)
}
if p.Descending {
s += " DESC"
}
parts = append(parts, s)
}
return strings.Join(parts, ", ")
}
func tableStatement(s *models.Schema, t *models.Table) string {
name := t.Name
schema := ""
if s != nil {
name = fmt.Sprintf("%s.%s", quote(s.Name), quote(t.Name))
} else {
name = quote(name)
schema = s.Name
}
cols := make([]*models.Column, 0, len(t.Columns))
for _, c := range t.Columns {
@@ -118,76 +223,196 @@ func tableStatement(s *models.Schema, t *models.Table) string {
defs := []string{}
pk := []string{}
for _, c := range cols {
d := fmt.Sprintf(" %s %s", quote(c.Name), mariadb.ConvertCanonicalToMariaDB(c.Type))
if c.Length > 0 && strings.EqualFold(c.Type, "string") {
d = fmt.Sprintf(" %s VARCHAR(%d)", quote(c.Name), c.Length)
d := " " + quote(c.Name) + " " + columnType(c)
if charset := metaString(c.Metadata, "mysql_charset"); charset != "" && c.Collation != metaString(t.Metadata, "mysql_collation") {
d += " CHARACTER SET " + quote(charset)
}
if c.Collation != "" && c.Collation != metaString(t.Metadata, "mysql_collation") {
d += " COLLATE " + quote(c.Collation)
}
if c.Generated {
d += " GENERATED ALWAYS AS (" + c.GenerationExpression + ")"
if strings.EqualFold(metaString(c.Metadata, "mysql_generation_storage"), "STORED") {
d += " STORED"
} else {
d += " VIRTUAL"
}
}
if c.NotNull {
d += " NOT NULL"
} else if !c.Generated {
d += " NULL"
}
if c.AutoIncrement {
d += " AUTO_INCREMENT"
}
if c.Default != nil {
d += fmt.Sprintf(" DEFAULT %s", writers.QuoteDefaultValue(fmt.Sprint(c.Default), c.Type))
if c.Default != nil && !c.Generated {
d += " DEFAULT " + defaultSQL(c)
}
if upd := metaString(c.Metadata, "mysql_on_update"); upd != "" {
d += " ON UPDATE " + upd
}
if metaBool(c.Metadata, "mysql_invisible", false) {
d += " INVISIBLE"
}
if c.Comment != "" {
d += " COMMENT " + literal(c.Comment)
}
defs = append(defs, d)
if c.IsPrimaryKey {
pk = append(pk, quote(c.Name))
pk = append(pk, c.Name)
}
}
constraintNames := make([]string, 0, len(t.Constraints))
for name := range t.Constraints {
constraintNames = append(constraintNames, name)
}
sort.Strings(constraintNames)
for _, name := range constraintNames {
for _, name := range keys(t.Constraints) {
c := t.Constraints[name]
if c.Type == models.PrimaryKeyConstraint {
pk = nil
for _, n := range c.Columns {
pk = append(pk, quote(n))
}
pk = c.Columns
break
}
}
if len(pk) > 0 {
defs = append(defs, " PRIMARY KEY ("+strings.Join(pk, ", ")+")")
}
for _, name := range constraintNames {
c := t.Constraints[name]
if c.Type == models.UniqueConstraint {
defs = append(defs, fmt.Sprintf(" CONSTRAINT %s UNIQUE (%s)", quote(c.Name), quoted(c.Columns)))
var primaryIndex *models.Index
for name, i := range t.Indexes {
if strings.EqualFold(name, "PRIMARY") {
primaryIndex = i
break
}
}
return fmt.Sprintf("CREATE TABLE IF NOT EXISTS %s (\n%s\n) ENGINE=InnoDB;\n\n", name, strings.Join(defs, ",\n"))
if i := primaryIndex; i != nil {
defs = append(defs, " PRIMARY KEY ("+indexParts(i)+")")
} else if len(pk) > 0 {
defs = append(defs, " PRIMARY KEY ("+quoted(pk)+")")
}
for _, name := range keys(t.Constraints) {
c := t.Constraints[name]
switch c.Type {
case models.UniqueConstraint:
if t.Indexes[c.Name] == nil {
defs = append(defs, " CONSTRAINT "+quote(c.Name)+" UNIQUE ("+quoted(c.Columns)+")")
}
case models.CheckConstraint:
d := " CONSTRAINT " + quote(c.Name) + " CHECK (" + c.Expression + ")"
if !metaBool(c.Metadata, "mysql_enforced", true) {
d += " NOT ENFORCED"
}
defs = append(defs, d)
}
}
for _, name := range keys(t.Indexes) {
i := t.Indexes[name]
if strings.EqualFold(name, "PRIMARY") {
continue
}
method := strings.ToUpper(i.Type)
if method != "" && method != "BTREE" && method != "HASH" && method != "FULLTEXT" && method != "SPATIAL" {
continue
}
kind := "KEY"
if i.Unique {
kind = "UNIQUE KEY"
}
if method == "FULLTEXT" || method == "SPATIAL" {
kind = method + " KEY"
}
d := " " + kind + " " + quote(i.Name) + " (" + indexParts(i) + ")"
if method == "HASH" {
d += " USING " + method
}
if !metaBool(i.Metadata, "mysql_visible", true) {
d += " INVISIBLE"
}
if i.Comment != "" {
d += " COMMENT " + literal(i.Comment)
}
defs = append(defs, d)
}
engine := metaString(t.Metadata, "mysql_engine")
if engine == "" {
engine = "InnoDB"
}
tail := " ENGINE=" + quote(engine)
if engine == "InnoDB" {
tail = " ENGINE=InnoDB"
}
if coll := metaString(t.Metadata, "mysql_collation"); coll != "" {
tail += " COLLATE=" + quote(coll)
}
if auto := metaString(t.Metadata, "mysql_auto_increment"); auto != "" {
if _, err := strconv.ParseUint(auto, 10, 64); err == nil {
tail += " AUTO_INCREMENT=" + auto
}
}
if t.Comment != "" {
tail += " COMMENT=" + literal(t.Comment)
}
return "CREATE TABLE IF NOT EXISTS " + qualified(schema, t.Name) + " (\n" + strings.Join(defs, ",\n") + "\n)" + tail + ";\n\n"
}
func databaseStatements(db *models.Database) []string {
var statements []string
for _, schema := range db.Schemas {
for _, table := range schema.Tables {
statements = append(statements, tableStatement(schema, table))
for _, s := range db.Schemas {
for _, t := range s.Tables {
statements = append(statements, tableStatement(s, t))
}
}
// References are added after all tables, supporting cycles and forward references.
for _, s := range db.Schemas {
for _, t := range s.Tables {
for _, name := range keys(t.Constraints) {
c := t.Constraints[name]
if c.Type != models.ForeignKeyConstraint {
continue
}
rs := c.ReferencedSchema
if rs == "" {
rs = s.Name
}
stmt := "ALTER TABLE " + qualified(s.Name, t.Name) + " ADD CONSTRAINT " + quote(c.Name) + " FOREIGN KEY (" + quoted(c.Columns) + ") REFERENCES " + qualified(rs, c.ReferencedTable) + " (" + quoted(c.ReferencedColumns) + ")"
if validAction(c.OnDelete) && c.OnDelete != "" {
stmt += " ON DELETE " + strings.ToUpper(c.OnDelete)
}
if validAction(c.OnUpdate) && c.OnUpdate != "" {
stmt += " ON UPDATE " + strings.ToUpper(c.OnUpdate)
}
statements = append(statements, stmt+";\n\n")
}
}
}
return statements
}
func (w *Writer) execute(dbm *models.Database, conn string) error {
statements := databaseStatements(dbm)
func (w *Writer) execute(statements []string, conn string) error {
if w.options.AtomicExecution {
return fmt.Errorf("MySQL DDL implicitly commits; atomic execution is unsupported")
}
db, err := sql.Open("mysql", conn)
if err != nil {
return fmt.Errorf("failed to connect: %w", err)
}
defer db.Close()
for _, stmt := range statements {
if _, err := db.ExecContext(context.Background(), stmt); err != nil {
return fmt.Errorf("failed to execute SQL: %w", err)
session, err := db.Conn(context.Background())
if err != nil {
return fmt.Errorf("failed to execute SQL: %w", err)
}
defer session.Close()
var mode string
if err := session.QueryRowContext(context.Background(), "SELECT @@SESSION.sql_mode").Scan(&mode); err != nil {
return fmt.Errorf("read MySQL SQL mode: %w", err)
}
if mode = unsupportedMode(mode); mode != "" {
return fmt.Errorf("MySQL execution requires standard quotes and backslash escaping; %s is enabled", mode)
}
var failures []string
for n, stmt := range statements {
if _, err := session.ExecContext(context.Background(), stmt); err != nil {
failures = append(failures, fmt.Sprintf("statement %d: %v", n+1, err))
if !w.options.ContinueOnError {
break
}
}
}
if len(failures) > 0 {
return fmt.Errorf("failed to execute SQL: %s", strings.Join(failures, "; "))
}
return nil
}
func quote(s string) string { return "`" + strings.ReplaceAll(s, "`", "``") + "`" }
func quoted(xs []string) string {
out := make([]string, len(xs))
@@ -196,3 +421,21 @@ func quoted(xs []string) string {
}
return strings.Join(out, ", ")
}
func validAction(action string) bool {
switch strings.ToUpper(action) {
case "", "CASCADE", "SET NULL", "RESTRICT", "NO ACTION":
return true
}
return false
}
func unsupportedMode(mode string) string {
var modes []string
for _, unsupported := range []string{"NO_BACKSLASH_ESCAPES", "ANSI_QUOTES"} {
if strings.Contains(strings.ToUpper(mode), unsupported) {
modes = append(modes, unsupported)
}
}
return strings.Join(modes, ", ")
}
+5
View File
@@ -56,6 +56,11 @@ const (
// WriterOptions contains common options for writers
type WriterOptions struct {
// StrictConversion rejects known lossy MySQL conversions before output or execution.
StrictConversion bool
// Warning receives conversion diagnostics. MySQL writes warnings to stderr when nil.
Warning func(string)
// OutputPath is the path where the output should be written
OutputPath string