wdevs/relspecgo
5
1
Fork 0
Code Issues 3 Pull Requests 1 Actions Packages Releases 87 Wiki Activity

fix: address audit execution failures and unsafe type conversions

Browse Source
This commit is contained in:
warkanum
2026-10-12 19:53:19 +02:00
parent bfe9852826
commit 2fa48335d3
29 changed files with 1630 additions and 421 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/writers/pgsql/writer.go
+121 -164
View File
@@ -2,7 +2,6 @@ package pgsql
import (
"context"
"encoding/json"
"fmt"
"io"
"os"
@@ -12,6 +11,8 @@ import (
"sync"
"time"
"github.com/jackc/pgx/v5"
"git.warky.dev/wdevs/relspecgo/pkg/models"
"git.warky.dev/wdevs/relspecgo/pkg/pgsql"
"git.warky.dev/wdevs/relspecgo/pkg/readers"
@@ -27,36 +28,12 @@ type Writer struct {
executor *TemplateExecutor
}
// ExecutionReport tracks the execution status of SQL statements
type ExecutionReport struct {
TotalStatements int `json:"total_statements"`
ExecutedStatements int `json:"executed_statements"`
FailedStatements int `json:"failed_statements"`
Schemas []SchemaReport `json:"schemas"`
Errors []ExecutionError `json:"errors,omitempty"`
StartTime string `json:"start_time"`
EndTime string `json:"end_time"`
}
// SchemaReport tracks execution per schema
type SchemaReport struct {
Name string `json:"name"`
Tables []TableReport `json:"tables"`
}
// TableReport tracks execution per table
type TableReport struct {
Name string `json:"name"`
Created bool `json:"created"`
Error string `json:"error,omitempty"`
}
// ExecutionError represents a failed statement
type ExecutionError struct {
StatementNumber int `json:"statement_number"`
Statement string `json:"statement"`
Error string `json:"error"`
}
// Shared execution results retain the PostgreSQL package's public type names.
type ExecutionReport = writers.ExecutionReport
type ExecutionFailure = writers.ExecutionFailure
type SchemaReport = writers.SchemaReport
type TableReport = writers.TableReport
type ExecutionError = writers.ExecutionError
// NewWriter creates a new PostgreSQL SQL writer
func NewWriter(options *writers.WriterOptions) *Writer {
@@ -109,9 +86,20 @@ func (w *Writer) WriteDatabase(db *models.Database) error {
}
fmt.Fprintf(w.writer, "\n")
for _, schema := range db.Schemas {
preflights, err := w.generateTypePreflights(schema)
if err != nil {
return err
}
for _, stmt := range preflights {
if _, err := fmt.Fprintln(w.writer, stmt); err != nil {
return err
}
}
}
// Process each schema in the database
for _, schema := range db.Schemas {
if err := w.WriteSchema(schema); err != nil {
if err := w.writeSchema(schema); err != nil {
return fmt.Errorf("failed to write schema %s: %w", schema.Name, err)
}
}
@@ -129,9 +117,16 @@ func (w *Writer) GenerateDatabaseStatements(db *models.Database) ([]string, erro
statements = append(statements, fmt.Sprintf("-- Database: %s", db.Name))
statements = append(statements, "-- Generated by RelSpec")
for _, schema := range db.Schemas {
preflights, err := w.generateTypePreflights(schema)
if err != nil {
return nil, err
}
statements = append(statements, preflights...)
}
// Process each schema in the database
for _, schema := range db.Schemas {
schemaStatements, err := w.GenerateSchemaStatements(schema)
schemaStatements, err := w.generateSchemaStatements(schema)
if err != nil {
return nil, fmt.Errorf("failed to generate statements for schema %s: %w", schema.Name, err)
}
@@ -187,6 +182,15 @@ func (w *Writer) primaryKeySetvalStatement(schema *models.Schema, table *models.
// GenerateSchemaStatements generates SQL statements as a list for a single schema
func (w *Writer) GenerateSchemaStatements(schema *models.Schema) ([]string, error) {
preflights, err := w.generateTypePreflights(schema)
if err != nil {
return nil, err
}
statements, err := w.generateSchemaStatements(schema)
return append(preflights, statements...), err
}
func (w *Writer) generateSchemaStatements(schema *models.Schema) ([]string, error) {
statements := []string{}
if err := w.checkDirectives(schema); err != nil {
@@ -510,15 +514,11 @@ func (w *Writer) GenerateAlterColumnTypeStatements(schema *models.Schema) ([]str
// rather than risk an ALTER COLUMN TYPE Postgres won't accept cleanly.
continue
}
targetType := effectiveAlterColumnSQLType(col)
stmt, err := w.executor.ExecuteAlterColumnTypeWithCheck(AlterColumnTypeWithCheckData{
SchemaName: schema.Name,
TableName: table.Name,
ColumnName: col.Name,
NewType: targetType,
EquivalentTypes: equivalentTypeListSQL(targetType),
UsingExpr: buildAlterColumnUsingExpression(col.Name, targetType),
})
data, err := conversionData(schema.Name, table.Name, col)
if err != nil {
return nil, err
}
stmt, err := w.executor.ExecuteAlterColumnTypeWithCheck(data)
if err != nil {
return nil, fmt.Errorf("failed to generate alter column type for %s.%s.%s: %w", schema.Name, table.Name, col.Name, err)
}
@@ -774,6 +774,22 @@ func (w *Writer) WriteSchema(schema *models.Schema) error {
if w.writer == nil {
w.writer = os.Stdout
}
preflights, err := w.generateTypePreflights(schema)
if err != nil {
return err
}
for _, stmt := range preflights {
if _, err := fmt.Fprintln(w.writer, stmt); err != nil {
return err
}
}
return w.writeSchema(schema)
}
func (w *Writer) writeSchema(schema *models.Schema) error {
if w.writer == nil {
w.writer = os.Stdout
}
if err := w.checkDirectives(schema); err != nil {
return err
@@ -2054,7 +2070,8 @@ func extractSequenceName(defaultExpr string) string {
func (w *Writer) executeDatabaseSQL(db *models.Database, connString string) error {
// Initialize execution report
w.executionReport = &ExecutionReport{
StartTime: getCurrentTimestamp(),
Dialect: "PostgreSQL",
StartTime: time.Now().Format(time.RFC3339Nano),
Schemas: make([]SchemaReport, 0),
Errors: make([]ExecutionError, 0),
}
@@ -2067,7 +2084,8 @@ func (w *Writer) executeDatabaseSQL(db *models.Database, connString string) erro
} else {
statements = diffStatements
if len(statements) == 0 {
w.executionReport.EndTime = getCurrentTimestamp()
w.executionReport.EndTime = time.Now().Format(time.RFC3339Nano)
w.executionReport.Outcome = "no_changes"
fmt.Fprintln(os.Stderr, "✓ Database is already up to date; nothing to execute")
return w.finishReport()
}
@@ -2179,12 +2197,17 @@ func (w *Writer) diffStatements(model, current *models.Database) ([]string, erro
return nil, fmt.Errorf("failed to generate migration scripts: %w", err)
}
var preflights []string
lastSchema := ""
for _, script := range scripts {
body := strings.TrimSpace(script.Body)
if body == "" {
continue
}
if script.ObjectType == "preflight column type" {
preflights = append(preflights, body)
continue
}
if script.Schema != "" && script.Schema != lastSchema {
statements = append(statements, fmt.Sprintf("-- Schema: %s", script.Schema))
lastSchema = script.Schema
@@ -2192,150 +2215,84 @@ func (w *Writer) diffStatements(model, current *models.Database) ([]string, erro
statements = append(statements, body)
}
statements = append(statements, setvalStatements...)
statements = append(preflights, statements...)
return statements, nil
}
// executeStatements runs statements one by one against the database and writes the report.
func (w *Writer) executeStatements(statements []string, connString string) error {
w.executionReport.TotalStatements = len(statements)
// Connect to database
fmt.Fprintln(os.Stderr, " → Connecting to PostgreSQL output database...")
ctx := context.Background()
conn, err := pgsql.Connect(ctx, connString, "writer-pgsql")
if err != nil {
return fmt.Errorf("failed to connect to database: %w", err)
err = writers.FailExecution(w.options, "PostgreSQL", writers.StatementOperations(statements), fmt.Errorf("failed to connect to database: %w", err))
w.executionReport = w.options.Metadata["execution_report"].(*ExecutionReport)
return err
}
defer conn.Close(ctx)
fmt.Fprintln(os.Stderr, " → Connected. Executing statements...")
// Track schemas and tables
schemaMap := make(map[string]*SchemaReport)
w.executionReport.Dialect = "PostgreSQL"
var tx pgx.Tx
exec := func(ctx context.Context, i int, stmt string) error {
fmt.Fprintf(os.Stderr, "Executing statement %d/%d [%s] %s...\n", i+1, len(statements), detectStatementType(stmt), extractStatementContext(stmt))
if tx != nil {
_, err := tx.Exec(ctx, stmt)
return err
}
_, err := conn.Exec(ctx, stmt)
return err
}
backend := writers.ExecutionBackend{
Exec: exec,
Begin: func(ctx context.Context) error { var err error; tx, err = conn.Begin(ctx); return err },
Commit: func(ctx context.Context) error { return tx.Commit(ctx) },
Rollback: func(ctx context.Context) error { return tx.Rollback(ctx) },
}
schemaNames := make([]string, len(statements))
currentSchema := ""
// Execute each statement
for i, stmt := range statements {
stmtTrimmed := strings.TrimSpace(stmt)
// Skip comments
if strings.HasPrefix(stmtTrimmed, "--") {
// Check if this is a schema comment to track schema changes
if strings.Contains(stmtTrimmed, "Schema:") {
parts := strings.Split(stmtTrimmed, "Schema:")
if len(parts) > 1 {
currentSchema = strings.TrimSpace(parts[1])
if _, exists := schemaMap[currentSchema]; !exists {
schemaReport := SchemaReport{
Name: currentSchema,
Tables: make([]TableReport, 0),
}
schemaMap[currentSchema] = &schemaReport
}
}
}
continue
}
// Skip empty statements
if stmtTrimmed == "" {
continue
}
stmtType := detectStatementType(stmtTrimmed)
stmtCtx := extractStatementContext(stmtTrimmed)
if stmtCtx != "" {
fmt.Fprintf(os.Stderr, "Executing statement %d/%d [%s] %s...\n", i+1, len(statements), stmtType, stmtCtx)
} else {
fmt.Fprintf(os.Stderr, "Executing statement %d/%d [%s]...\n", i+1, len(statements), stmtType)
}
_, execErr := conn.Exec(ctx, stmt)
if execErr != nil {
w.executionReport.FailedStatements++
execError := ExecutionError{
StatementNumber: i + 1,
Statement: truncateStatement(stmt),
Error: execErr.Error(),
}
w.executionReport.Errors = append(w.executionReport.Errors, execError)
// Track table creation failure
if strings.Contains(strings.ToUpper(stmtTrimmed), "CREATE TABLE") && currentSchema != "" {
tableName := extractTableNameFromCreate(stmtTrimmed)
if tableName != "" && schemaMap[currentSchema] != nil {
schemaMap[currentSchema].Tables = append(schemaMap[currentSchema].Tables, TableReport{
Name: tableName,
Created: false,
Error: execErr.Error(),
})
}
}
// Continue with next statement instead of failing completely
fmt.Fprintf(os.Stderr, "⚠ Warning: Statement %d failed: %v\n", i+1, execErr)
continue
}
w.executionReport.ExecutedStatements++
// Track successful table creation
if strings.Contains(strings.ToUpper(stmtTrimmed), "CREATE TABLE") && currentSchema != "" {
tableName := extractTableNameFromCreate(stmtTrimmed)
if tableName != "" && schemaMap[currentSchema] != nil {
schemaMap[currentSchema].Tables = append(schemaMap[currentSchema].Tables, TableReport{
Name: tableName,
Created: true,
})
}
if strings.HasPrefix(strings.TrimSpace(stmt), "-- Schema:") {
currentSchema = strings.TrimSpace(strings.TrimPrefix(strings.TrimSpace(stmt), "-- Schema:"))
}
schemaNames[i] = currentSchema
}
// Convert schema map to slice
for _, schemaReport := range schemaMap {
w.executionReport.Schemas = append(w.executionReport.Schemas, *schemaReport)
schemaIndexes := make(map[string]int)
// Populate per-schema details before finalizing the shared report.
backend.Exec = func(ctx context.Context, i int, stmt string) error {
err := exec(ctx, i, stmt)
if strings.Contains(strings.ToUpper(stmt), "CREATE TABLE") {
schemaName := schemaNames[i]
if schemaName != "" {
idx, exists := schemaIndexes[schemaName]
if !exists {
idx = -1
}
if idx < 0 {
w.executionReport.Schemas = append(w.executionReport.Schemas, SchemaReport{Name: schemaName, Tables: []TableReport{}})
idx = len(w.executionReport.Schemas) - 1
schemaIndexes[schemaName] = idx
}
table := TableReport{Name: extractTableNameFromCreate(stmt), Created: err == nil}
if err != nil {
table.Error = err.Error()
}
w.executionReport.Schemas[idx].Tables = append(w.executionReport.Schemas[idx].Tables, table)
}
}
return err
}
w.executionReport.EndTime = getCurrentTimestamp()
return w.finishReport()
return writers.RunExecution(ctx, w.options, w.executionReport, writers.StatementOperations(statements), backend)
}
// finishReport writes the optional report file and prints the execution summary.
func (w *Writer) finishReport() error {
// Write report if path is specified
if reportPath, ok := w.options.Metadata["report_path"].(string); ok && reportPath != "" {
if err := w.writeReport(reportPath); err != nil {
fmt.Fprintf(os.Stderr, "⚠ Warning: Failed to write report: %v\n", err)
} else {
fmt.Fprintf(os.Stderr, "✓ Report written to: %s\n", reportPath)
}
}
var err error
if w.executionReport.FailedStatements > 0 {
fmt.Fprintf(os.Stderr, "⚠ Completed with %d errors out of %d statements\n",
w.executionReport.FailedStatements, w.executionReport.TotalStatements)
} else {
fmt.Fprintf(os.Stderr, "✓ Successfully executed %d statements\n", w.executionReport.ExecutedStatements)
err = &ExecutionFailure{Report: w.executionReport}
}
return nil
return writers.FinishExecution(w.options, w.executionReport, err)
}
// writeReport writes the execution report to a JSON file
func (w *Writer) writeReport(reportPath string) error {
file, err := os.Create(reportPath)
if err != nil {
return fmt.Errorf("failed to create report file: %w", err)
}
defer file.Close()
encoder := json.NewEncoder(file)
encoder.SetIndent("", " ")
if err := encoder.Encode(w.executionReport); err != nil {
return fmt.Errorf("failed to encode report: %w", err)
}
return nil
return writers.WriteExecutionReport(reportPath, w.executionReport)
}
// extractTableNameFromCreate extracts table name from CREATE TABLE statement