sql writer
This commit is contained in:
696
pkg/writers/pgsql/TEMPLATES.md
Normal file
696
pkg/writers/pgsql/TEMPLATES.md
Normal file
@@ -0,0 +1,696 @@
|
||||
# PostgreSQL Migration Templates
|
||||
|
||||
## Overview
|
||||
|
||||
The PostgreSQL migration writer uses Go text templates to generate SQL, making the code much more maintainable and customizable than hardcoded string concatenation.
|
||||
|
||||
## Architecture
|
||||
|
||||
```
|
||||
pkg/writers/pgsql/
|
||||
├── templates/ # Template files
|
||||
│ ├── create_table.tmpl # CREATE TABLE
|
||||
│ ├── add_column.tmpl # ALTER TABLE ADD COLUMN
|
||||
│ ├── alter_column_type.tmpl # ALTER TABLE ALTER COLUMN TYPE
|
||||
│ ├── alter_column_default.tmpl # ALTER TABLE ALTER COLUMN DEFAULT
|
||||
│ ├── create_primary_key.tmpl # ADD CONSTRAINT PRIMARY KEY
|
||||
│ ├── create_index.tmpl # CREATE INDEX
|
||||
│ ├── create_foreign_key.tmpl # ADD CONSTRAINT FOREIGN KEY
|
||||
│ ├── drop_constraint.tmpl # DROP CONSTRAINT
|
||||
│ ├── drop_index.tmpl # DROP INDEX
|
||||
│ ├── comment_table.tmpl # COMMENT ON TABLE
|
||||
│ ├── comment_column.tmpl # COMMENT ON COLUMN
|
||||
│ ├── audit_tables.tmpl # CREATE audit tables
|
||||
│ ├── audit_function.tmpl # CREATE audit function
|
||||
│ └── audit_trigger.tmpl # CREATE audit trigger
|
||||
├── templates.go # Template executor and data structures
|
||||
└── migration_writer_templated.go # Templated migration writer
|
||||
```
|
||||
|
||||
## Using Templates
|
||||
|
||||
### Basic Usage
|
||||
|
||||
```go
|
||||
// Create template executor
|
||||
executor, err := pgsql.NewTemplateExecutor()
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
// Prepare data
|
||||
data := pgsql.CreateTableData{
|
||||
SchemaName: "public",
|
||||
TableName: "users",
|
||||
Columns: []pgsql.ColumnData{
|
||||
{Name: "id", Type: "integer", NotNull: true},
|
||||
{Name: "name", Type: "text"},
|
||||
},
|
||||
}
|
||||
|
||||
// Execute template
|
||||
sql, err := executor.ExecuteCreateTable(data)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
fmt.Println(sql)
|
||||
```
|
||||
|
||||
### Using Templated Migration Writer
|
||||
|
||||
```go
|
||||
// Create templated migration writer
|
||||
writer, err := pgsql.NewTemplatedMigrationWriter(&writers.WriterOptions{
|
||||
OutputPath: "migration.sql",
|
||||
})
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
// Generate migration (uses templates internally)
|
||||
err = writer.WriteMigration(modelDB, currentDB)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
```
|
||||
|
||||
## Template Data Structures
|
||||
|
||||
### CreateTableData
|
||||
|
||||
For `create_table.tmpl`:
|
||||
|
||||
```go
|
||||
type CreateTableData struct {
|
||||
SchemaName string
|
||||
TableName string
|
||||
Columns []ColumnData
|
||||
}
|
||||
|
||||
type ColumnData struct {
|
||||
Name string
|
||||
Type string
|
||||
Default string
|
||||
NotNull bool
|
||||
}
|
||||
```
|
||||
|
||||
Example:
|
||||
```go
|
||||
data := CreateTableData{
|
||||
SchemaName: "public",
|
||||
TableName: "products",
|
||||
Columns: []ColumnData{
|
||||
{Name: "id", Type: "serial", NotNull: true},
|
||||
{Name: "name", Type: "text", NotNull: true},
|
||||
{Name: "price", Type: "numeric(10,2)", Default: "0.00"},
|
||||
},
|
||||
}
|
||||
```
|
||||
|
||||
### AddColumnData
|
||||
|
||||
For `add_column.tmpl`:
|
||||
|
||||
```go
|
||||
type AddColumnData struct {
|
||||
SchemaName string
|
||||
TableName string
|
||||
ColumnName string
|
||||
ColumnType string
|
||||
Default string
|
||||
NotNull bool
|
||||
}
|
||||
```
|
||||
|
||||
### CreateIndexData
|
||||
|
||||
For `create_index.tmpl`:
|
||||
|
||||
```go
|
||||
type CreateIndexData struct {
|
||||
SchemaName string
|
||||
TableName string
|
||||
IndexName string
|
||||
IndexType string // btree, hash, gin, gist
|
||||
Columns string // comma-separated
|
||||
Unique bool
|
||||
}
|
||||
```
|
||||
|
||||
### CreateForeignKeyData
|
||||
|
||||
For `create_foreign_key.tmpl`:
|
||||
|
||||
```go
|
||||
type CreateForeignKeyData struct {
|
||||
SchemaName string
|
||||
TableName string
|
||||
ConstraintName string
|
||||
SourceColumns string // comma-separated
|
||||
TargetSchema string
|
||||
TargetTable string
|
||||
TargetColumns string // comma-separated
|
||||
OnDelete string // CASCADE, SET NULL, etc.
|
||||
OnUpdate string
|
||||
}
|
||||
```
|
||||
|
||||
### AuditFunctionData
|
||||
|
||||
For `audit_function.tmpl`:
|
||||
|
||||
```go
|
||||
type AuditFunctionData struct {
|
||||
SchemaName string
|
||||
FunctionName string
|
||||
TableName string
|
||||
TablePrefix string
|
||||
PrimaryKey string
|
||||
AuditSchema string
|
||||
UserFunction string
|
||||
AuditInsert bool
|
||||
AuditUpdate bool
|
||||
AuditDelete bool
|
||||
UpdateCondition string
|
||||
UpdateColumns []AuditColumnData
|
||||
DeleteColumns []AuditColumnData
|
||||
}
|
||||
|
||||
type AuditColumnData struct {
|
||||
Name string
|
||||
OldValue string // SQL expression for old value
|
||||
NewValue string // SQL expression for new value
|
||||
}
|
||||
```
|
||||
|
||||
## Customizing Templates
|
||||
|
||||
### Modifying Existing Templates
|
||||
|
||||
Templates are embedded in the binary but can be modified at compile time:
|
||||
|
||||
1. **Edit template file** in `pkg/writers/pgsql/templates/`:
|
||||
|
||||
```go
|
||||
// templates/create_table.tmpl
|
||||
CREATE TABLE IF NOT EXISTS {{.SchemaName}}.{{.TableName}} (
|
||||
{{- range $i, $col := .Columns}}
|
||||
{{- if $i}},{{end}}
|
||||
{{$col.Name}} {{$col.Type}}
|
||||
{{- if $col.Default}} DEFAULT {{$col.Default}}{{end}}
|
||||
{{- if $col.NotNull}} NOT NULL{{end}}
|
||||
{{- end}}
|
||||
);
|
||||
|
||||
-- Custom comment
|
||||
COMMENT ON TABLE {{.SchemaName}}.{{.TableName}} IS 'Auto-generated by RelSpec';
|
||||
```
|
||||
|
||||
2. **Rebuild** the application:
|
||||
|
||||
```bash
|
||||
go build ./cmd/relspec
|
||||
```
|
||||
|
||||
The new template is automatically embedded.
|
||||
|
||||
### Template Syntax Reference
|
||||
|
||||
#### Variables
|
||||
|
||||
```go
|
||||
{{.FieldName}} // Access field
|
||||
{{.SchemaName}} // String field
|
||||
{{.NotNull}} // Boolean field
|
||||
```
|
||||
|
||||
#### Conditionals
|
||||
|
||||
```go
|
||||
{{if .NotNull}}
|
||||
NOT NULL
|
||||
{{end}}
|
||||
|
||||
{{if .Default}}
|
||||
DEFAULT {{.Default}}
|
||||
{{else}}
|
||||
-- No default
|
||||
{{end}}
|
||||
```
|
||||
|
||||
#### Loops
|
||||
|
||||
```go
|
||||
{{range $i, $col := .Columns}}
|
||||
Column: {{$col.Name}} Type: {{$col.Type}}
|
||||
{{end}}
|
||||
```
|
||||
|
||||
#### Functions
|
||||
|
||||
```go
|
||||
{{if eq .Type "CASCADE"}}
|
||||
ON DELETE CASCADE
|
||||
{{end}}
|
||||
|
||||
{{join .Columns ", "}} // Join string slice
|
||||
```
|
||||
|
||||
### Creating New Templates
|
||||
|
||||
1. **Create template file** in `pkg/writers/pgsql/templates/`:
|
||||
|
||||
```go
|
||||
// templates/custom_operation.tmpl
|
||||
-- Custom operation for {{.TableName}}
|
||||
ALTER TABLE {{.SchemaName}}.{{.TableName}}
|
||||
{{.CustomOperation}};
|
||||
```
|
||||
|
||||
2. **Define data structure** in `templates.go`:
|
||||
|
||||
```go
|
||||
type CustomOperationData struct {
|
||||
SchemaName string
|
||||
TableName string
|
||||
CustomOperation string
|
||||
}
|
||||
```
|
||||
|
||||
3. **Add executor method** in `templates.go`:
|
||||
|
||||
```go
|
||||
func (te *TemplateExecutor) ExecuteCustomOperation(data CustomOperationData) (string, error) {
|
||||
var buf bytes.Buffer
|
||||
err := te.templates.ExecuteTemplate(&buf, "custom_operation.tmpl", data)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to execute custom_operation template: %w", err)
|
||||
}
|
||||
return buf.String(), nil
|
||||
}
|
||||
```
|
||||
|
||||
4. **Use in migration writer**:
|
||||
|
||||
```go
|
||||
sql, err := w.executor.ExecuteCustomOperation(CustomOperationData{
|
||||
SchemaName: "public",
|
||||
TableName: "users",
|
||||
CustomOperation: "ADD COLUMN custom_field text",
|
||||
})
|
||||
```
|
||||
|
||||
## Template Examples
|
||||
|
||||
### Example 1: Custom Table Creation
|
||||
|
||||
Modify `create_table.tmpl` to add table options:
|
||||
|
||||
```sql
|
||||
CREATE TABLE IF NOT EXISTS {{.SchemaName}}.{{.TableName}} (
|
||||
{{- range $i, $col := .Columns}}
|
||||
{{- if $i}},{{end}}
|
||||
{{$col.Name}} {{$col.Type}}
|
||||
{{- if $col.Default}} DEFAULT {{$col.Default}}{{end}}
|
||||
{{- if $col.NotNull}} NOT NULL{{end}}
|
||||
{{- end}}
|
||||
) WITH (fillfactor = 90);
|
||||
|
||||
-- Add automatic comment
|
||||
COMMENT ON TABLE {{.SchemaName}}.{{.TableName}}
|
||||
IS 'Created: {{.CreatedDate}} | Version: {{.Version}}';
|
||||
```
|
||||
|
||||
### Example 2: Custom Index with WHERE Clause
|
||||
|
||||
Add to `create_index.tmpl`:
|
||||
|
||||
```sql
|
||||
CREATE {{if .Unique}}UNIQUE {{end}}INDEX IF NOT EXISTS {{.IndexName}}
|
||||
ON {{.SchemaName}}.{{.TableName}}
|
||||
USING {{.IndexType}} ({{.Columns}})
|
||||
{{- if .Where}}
|
||||
WHERE {{.Where}}
|
||||
{{- end}}
|
||||
{{- if .Include}}
|
||||
INCLUDE ({{.Include}})
|
||||
{{- end}};
|
||||
```
|
||||
|
||||
Update data structure:
|
||||
|
||||
```go
|
||||
type CreateIndexData struct {
|
||||
SchemaName string
|
||||
TableName string
|
||||
IndexName string
|
||||
IndexType string
|
||||
Columns string
|
||||
Unique bool
|
||||
Where string // New field for partial indexes
|
||||
Include string // New field for covering indexes
|
||||
}
|
||||
```
|
||||
|
||||
### Example 3: Enhanced Audit Function
|
||||
|
||||
Modify `audit_function.tmpl` to add custom logging:
|
||||
|
||||
```sql
|
||||
CREATE OR REPLACE FUNCTION {{.SchemaName}}.{{.FunctionName}}()
|
||||
RETURNS trigger AS
|
||||
$body$
|
||||
DECLARE
|
||||
m_funcname text = '{{.FunctionName}}';
|
||||
m_user text;
|
||||
m_atevent integer;
|
||||
m_application_name text;
|
||||
BEGIN
|
||||
-- Get current user and application
|
||||
m_user := {{.UserFunction}}::text;
|
||||
m_application_name := current_setting('application_name', true);
|
||||
|
||||
-- Custom logging
|
||||
RAISE NOTICE 'Audit: % on %.% by % from %',
|
||||
TG_OP, TG_TABLE_SCHEMA, TG_TABLE_NAME, m_user, m_application_name;
|
||||
|
||||
-- Rest of function...
|
||||
...
|
||||
```
|
||||
|
||||
## Best Practices
|
||||
|
||||
### 1. Keep Templates Simple
|
||||
|
||||
Templates should focus on SQL generation. Complex logic belongs in Go code:
|
||||
|
||||
**Good:**
|
||||
```go
|
||||
// In Go code
|
||||
columns := buildColumnList(table)
|
||||
|
||||
// In template
|
||||
{{range .Columns}}
|
||||
{{.Name}} {{.Type}}
|
||||
{{end}}
|
||||
```
|
||||
|
||||
**Bad:**
|
||||
```go
|
||||
// Don't do complex transformations in templates
|
||||
{{range .Columns}}
|
||||
{{if eq .Type "integer"}}
|
||||
{{.Name}} serial
|
||||
{{else}}
|
||||
{{.Name}} {{.Type}}
|
||||
{{end}}
|
||||
{{end}}
|
||||
```
|
||||
|
||||
### 2. Use Descriptive Field Names
|
||||
|
||||
```go
|
||||
// Good
|
||||
type CreateTableData struct {
|
||||
SchemaName string
|
||||
TableName string
|
||||
}
|
||||
|
||||
// Bad
|
||||
type CreateTableData struct {
|
||||
S string // What is S?
|
||||
T string // What is T?
|
||||
}
|
||||
```
|
||||
|
||||
### 3. Document Template Data
|
||||
|
||||
Always document what data a template expects:
|
||||
|
||||
```go
|
||||
// CreateTableData contains data for create table template.
|
||||
// Used by templates/create_table.tmpl
|
||||
type CreateTableData struct {
|
||||
SchemaName string // Schema where table will be created
|
||||
TableName string // Name of the table
|
||||
Columns []ColumnData // List of columns to create
|
||||
}
|
||||
```
|
||||
|
||||
### 4. Handle SQL Injection
|
||||
|
||||
Always escape user input:
|
||||
|
||||
```go
|
||||
// In Go code - escape before passing to template
|
||||
data := CommentTableData{
|
||||
SchemaName: schema,
|
||||
TableName: table,
|
||||
Comment: escapeQuote(userComment), // Escape quotes
|
||||
}
|
||||
```
|
||||
|
||||
### 5. Test Templates Thoroughly
|
||||
|
||||
```go
|
||||
func TestTemplate_CreateTable(t *testing.T) {
|
||||
executor, _ := NewTemplateExecutor()
|
||||
|
||||
data := CreateTableData{
|
||||
SchemaName: "public",
|
||||
TableName: "test",
|
||||
Columns: []ColumnData{{Name: "id", Type: "integer"}},
|
||||
}
|
||||
|
||||
sql, err := executor.ExecuteCreateTable(data)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Verify expected SQL patterns
|
||||
if !strings.Contains(sql, "CREATE TABLE") {
|
||||
t.Error("Missing CREATE TABLE")
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## Benefits of Template-Based Approach
|
||||
|
||||
### Maintainability
|
||||
|
||||
**Before (string concatenation):**
|
||||
```go
|
||||
sql := fmt.Sprintf(`CREATE TABLE %s.%s (
|
||||
%s %s%s%s
|
||||
);`, schema, table, col, typ,
|
||||
func() string {
|
||||
if def != "" {
|
||||
return " DEFAULT " + def
|
||||
}
|
||||
return ""
|
||||
}(),
|
||||
func() string {
|
||||
if notNull {
|
||||
return " NOT NULL"
|
||||
}
|
||||
return ""
|
||||
}(),
|
||||
)
|
||||
```
|
||||
|
||||
**After (templates):**
|
||||
```go
|
||||
sql, _ := executor.ExecuteCreateTable(CreateTableData{
|
||||
SchemaName: schema,
|
||||
TableName: table,
|
||||
Columns: columns,
|
||||
})
|
||||
```
|
||||
|
||||
### Customization
|
||||
|
||||
Users can modify templates without changing Go code:
|
||||
- Edit template file
|
||||
- Rebuild application
|
||||
- New SQL generation logic active
|
||||
|
||||
### Testing
|
||||
|
||||
Templates can be tested independently:
|
||||
```go
|
||||
func TestAuditTemplate(t *testing.T) {
|
||||
executor, _ := NewTemplateExecutor()
|
||||
|
||||
// Test with various data
|
||||
for _, testCase := range testCases {
|
||||
sql, err := executor.ExecuteAuditFunction(testCase.data)
|
||||
// Verify output
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### Readability
|
||||
|
||||
SQL templates are easier to read and review than Go string building code.
|
||||
|
||||
## Migration from Old Writer
|
||||
|
||||
To migrate from the old string-based writer to templates:
|
||||
|
||||
### Option 1: Use TemplatedMigrationWriter
|
||||
|
||||
```go
|
||||
// Old
|
||||
writer := pgsql.NewMigrationWriter(options)
|
||||
|
||||
// New
|
||||
writer, err := pgsql.NewTemplatedMigrationWriter(options)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
// Same interface
|
||||
writer.WriteMigration(model, current)
|
||||
```
|
||||
|
||||
### Option 2: Keep Both
|
||||
|
||||
Both writers are available:
|
||||
- `MigrationWriter` - Original string-based
|
||||
- `TemplatedMigrationWriter` - New template-based
|
||||
|
||||
Choose based on your needs.
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
### Template Not Found
|
||||
|
||||
```
|
||||
Error: template: "my_template.tmpl" not defined
|
||||
```
|
||||
|
||||
Solution: Ensure template file exists in `templates/` directory and rebuild.
|
||||
|
||||
### Template Execution Error
|
||||
|
||||
```
|
||||
Error: template: create_table.tmpl:5:10: executing "create_table.tmpl"
|
||||
at <.InvalidField>: can't evaluate field InvalidField
|
||||
```
|
||||
|
||||
Solution: Check data structure has all fields used in template.
|
||||
|
||||
### Embedded Files Not Updating
|
||||
|
||||
If template changes aren't reflected:
|
||||
|
||||
1. Clean build cache: `go clean -cache`
|
||||
2. Rebuild: `go build ./cmd/relspec`
|
||||
3. Verify template file is in `templates/` directory
|
||||
|
||||
## Custom Template Functions
|
||||
|
||||
RelSpec provides a comprehensive library of template functions for SQL generation:
|
||||
|
||||
### String Manipulation
|
||||
- `upper`, `lower` - Case conversion
|
||||
- `snake_case`, `camelCase` - Naming convention conversion
|
||||
- Usage: `{{upper .TableName}}` → `USERS`
|
||||
|
||||
### SQL Formatting
|
||||
- `indent(spaces, text)` - Indent text
|
||||
- `quote(string)` - Quote for SQL with escaping
|
||||
- `escape(string)` - Escape special characters
|
||||
- `safe_identifier(string)` - Make SQL-safe identifier
|
||||
- Usage: `{{quote "O'Brien"}}` → `'O''Brien'`
|
||||
|
||||
### Type Conversion
|
||||
- `goTypeToSQL(type)` - Convert Go type to PostgreSQL type
|
||||
- `sqlTypeToGo(type)` - Convert PostgreSQL type to Go type
|
||||
- `isNumeric(type)`, `isText(type)` - Type checking
|
||||
- Usage: `{{goTypeToSQL "int64"}}` → `bigint`
|
||||
|
||||
### Collection Helpers
|
||||
- `first(slice)`, `last(slice)` - Get elements
|
||||
- `join_with(slice, sep)` - Join with custom separator
|
||||
- Usage: `{{join_with .Columns ", "}}` → `id, name, email`
|
||||
|
||||
See [template_functions.go](template_functions.go) for full documentation.
|
||||
|
||||
## Template Inheritance and Composition
|
||||
|
||||
RelSpec supports Go template inheritance using `{{template}}` and `{{block}}`:
|
||||
|
||||
### Base Templates
|
||||
- `base_ddl.tmpl` - Common DDL patterns
|
||||
- `base_constraint.tmpl` - Constraint operations
|
||||
- `fragments.tmpl` - Reusable fragments
|
||||
|
||||
### Using Fragments
|
||||
```gotmpl
|
||||
{{/* Use predefined fragments */}}
|
||||
CREATE TABLE {{template "qualified_table" .}} (
|
||||
{{range .Columns}}
|
||||
{{template "column_definition" .}}
|
||||
{{end}}
|
||||
);
|
||||
```
|
||||
|
||||
### Template Blocks
|
||||
```gotmpl
|
||||
{{/* Define with override capability */}}
|
||||
{{define "table_options"}}
|
||||
) {{block "storage_options" .}}WITH (fillfactor = 90){{end}};
|
||||
{{end}}
|
||||
```
|
||||
|
||||
See [TEMPLATE_INHERITANCE.md](TEMPLATE_INHERITANCE.md) for detailed guide.
|
||||
|
||||
## Visual Template Editor
|
||||
|
||||
A VS Code extension is available for visual template editing:
|
||||
|
||||
### Features
|
||||
- **Live Preview** - See rendered SQL as you type
|
||||
- **IntelliSense** - Auto-completion for functions
|
||||
- **Validation** - Syntax checking and error highlighting
|
||||
- **Scaffolding** - Quick template creation
|
||||
- **Function Browser** - Browse available functions
|
||||
|
||||
### Installation
|
||||
```bash
|
||||
cd vscode-extension
|
||||
npm install
|
||||
npm run compile
|
||||
code .
|
||||
# Press F5 to launch
|
||||
```
|
||||
|
||||
See [vscode-extension/README.md](../../vscode-extension/README.md) for full documentation.
|
||||
|
||||
## Future Enhancements
|
||||
|
||||
Completed:
|
||||
- [x] Template inheritance/composition
|
||||
- [x] Custom template functions library
|
||||
- [x] Visual template editor (VS Code)
|
||||
|
||||
Potential future improvements:
|
||||
- [ ] Parameterized templates (load from config)
|
||||
- [ ] Template validation CLI tool
|
||||
- [ ] Template library/marketplace
|
||||
- [ ] Template versioning
|
||||
- [ ] Hot-reload during development
|
||||
|
||||
## Contributing Templates
|
||||
|
||||
When contributing new templates:
|
||||
|
||||
1. Place in `pkg/writers/pgsql/templates/`
|
||||
2. Use `.tmpl` extension
|
||||
3. Document data structure in `templates.go`
|
||||
4. Add executor method
|
||||
5. Write tests
|
||||
6. Update this documentation
|
||||
Reference in New Issue
Block a user