diff --git a/README.md b/README.md index c171dd4..2c1eb07 100644 --- a/README.md +++ b/README.md @@ -76,6 +76,7 @@ pgtidy config # print resolved config - Leading-comma lists (SELECT columns, function params) - Function params one-per-line; `LANGUAGE`, `SECURITY`, volatility each on own line - PL/pgSQL: `DECLARE` block vars 2-space indented; `BEGIN`/`END` at body level +- Procedural languages: bodies in `pl*` languages other than `plpgsql` (e.g. `plpython3u`, `plperl`), `c` and `internal` are left verbatim; `plpgsql` and `sql` bodies are formatted, and the function header is always formatted - Spaces around binary operators (`=`, `<>`, `||`, `:=`); no space before `(` or around `::`, `->`, `->>` --- diff --git a/docs/plan.md b/docs/plan.md index 6d7fdc0..487d7a7 100644 --- a/docs/plan.md +++ b/docs/plan.md @@ -67,6 +67,9 @@ to the intended style below. on own line; `AS` then `$$` on its own line; body; closing `$$;` on its own line. - PL/pgSQL: `DECLARE` alone, vars 2-space indented; `--Block--` comment markers preserved; `BEGIN`/`END` at body level; `IF/THEN/ELSIF/ELSE/END IF`, loops, `CASE` indent their bodies. +- Procedural languages: bodies in any `pl*` language other than `plpgsql` (`plpython3u`, + `plperl`, `pltcl`, …) and in `c` / `internal` are emitted verbatim; `plpgsql`, `sql` and routines with no `LANGUAGE` clause (treated as `sql`) are formatted. The + function header is always formatted. - Spacing: spaces around binary operators (`=`,`<>`,`||`,…) and `:=`; **no** space around `::`, `->`, `->>`, array `[...]`, or before a call's `(`. - Dollar-quote tags preserved verbatim (`$$`, `$S$`, `$Z$`, …). diff --git a/docs/todo.md b/docs/todo.md index 53e028a..59d1ffb 100644 --- a/docs/todo.md +++ b/docs/todo.md @@ -36,9 +36,9 @@ Legend: ✅ done · 🚧 in progress · ⬜ not started - `parser_test.go`: small round-trips, CreateFunction shape assertions, Raw fallback, and **corpus round-trip** — reconstructs all 4 files byte-for-byte; structures all 4 functions. - **Status:** all tests pass. -- _Still TODO (later): DML/other-DDL structuring (currently Raw) for full formatting._ +- _Note: DML/other-DDL still parse to `Raw`; they are formatted at token level by `pkg/format/dml.go`, not via structured CST nodes._ -### 🚧 PL/pgSQL body parser ← NEXT (the remaining V1 piece) +### ✅ PL/pgSQL body parser #### ✅ DECLARE section — `pkg/format/body.go` - `formatBody` splits the dollar-quote tag, calls `formatBodyInner`. @@ -243,6 +243,11 @@ Tests: `TestDMLInsertValues*`. ### ✅ Formatter — PL/pgSQL body settings (`pkg/format/body.go`) +- Language gate (`isPlpgsql` in `pkg/format/format.go`): bodies in a `pl*` language other than + `plpgsql` (`plpython3u`, `plperl`, …), `c` and `internal` are skipped and kept verbatim; `plpgsql`, `sql` and no + clause (treated as `sql`) are formatted. Header is always formatted. Test: + `TestNonPlpgsqlBodyVerbatim`. + - `plpgsql_max_blank_lines`: blank-line runs capped at the configured limit; default `1`. - `plpgsql_declare_align_type` + `plpgsql_declare_align_eq`: two-pass declare formatter measures name/type widths then pads for alignment; `writeDeclareAligned` helper. Both @@ -286,3 +291,27 @@ not implemented. ## Open risks - `go-pgquery` tracks PG17 (not PG18) — fine for lint; irrelevant to formatter path. - Leading-comma + one-per-line is a first-class style option, not an afterthought. + +### ✅ Literal & embedded-code safety (found via `temp/` corpus) + +- Multi-line quoted strings and dollar-quoted literals inside bodies are carried verbatim + (never re-indented); only the code outside them is scanned for parens / terminators / block + depth (`formatBodyStatements`, `cut`/`tail` handling). Same for top-level DML + (`litNL` sentinel in `dml.go`). +- Dollar-quoted literals whose content starts like code (`declare`/`begin` → PL/pgSQL block; + `select`/`insert`/`update`/`delete`/`with` → DML) are formatted recursively + (`formatEmbedded`); anything else (fragments, prose, other languages) stays verbatim. Literals + containing `format()` placeholders (`%s`, `%I`, `%L`, `%1$s`) are never touched, and a result + that fails `SemanticallyEqual`/`CommentsPreserved` is discarded. +- `DO LANGUAGE x $$…$$` keeps its `LANGUAGE` clause (it was being dropped). +- Code is never joined onto a line that ends in a `--` comment. +- A line ending in a literal or non-keyword no longer counts as ending in `EXCEPTION`/`THEN`/…. +- Comments between the last DECLARE variable and `BEGIN` are kept. +- Golden `testdata/corpus/test_mm_proc.pgsql` regenerated: nested dollar-quoted dynamic SQL is no + longer re-flowed. + +## Local corpus (`temp/`) + +`temp/` holds real-world routines (plpgsql, plpython3u, triggers) used for local testing only +(not committed). Run `pgtidy fmt` over it to find safety-gate failures. See "Language gate" above +for how non-plpgsql bodies are handled. diff --git a/pkg/format/body.go b/pkg/format/body.go index 78e1c1b..7152111 100644 --- a/pkg/format/body.go +++ b/pkg/format/body.go @@ -1,6 +1,9 @@ package format import ( + "git.warky.dev/wdevs/pgtidy/pkg/parser" + "regexp" + "sort" "strings" "git.warky.dev/wdevs/pgtidy/pkg/config" @@ -109,6 +112,16 @@ func formatBodyInner(inner string, st config.Style) string { // Format each variable declaration in the DECLARE section. formatDeclareVars(&b, sig[declareIdx+1:beginIdx], st) + // Comments between the last declaration and BEGIN live in BEGIN's leading + // trivia; keep them, one per line, at declaration indent. + for _, tr := range sig[beginIdx].Lead { + if tr.Kind == lexer.LineComment || tr.Kind == lexer.BlockComment { + b.WriteString(st.Indent) + b.WriteString(strings.TrimRight(strings.ReplaceAll(tr.Text, "\r", ""), " \t")) + b.WriteString(nl) + } + } + // Format the BEGIN…END block. b.WriteString(formatBodyStatements(inner[sig[beginIdx].Tok.Off:], st)) @@ -395,22 +408,57 @@ type bline struct { // 4. Blank-line counts from the original are preserved (capped by PlpgsqlMaxBlankLines). func formatBodyStatements(text string, st config.Style) string { nl := st.Newline + text = formatEmbeddedLiterals(text, st) normalised := strings.ReplaceAll(text, "\r\n", "\n") rawLines := strings.Split(normalised, "\n") - // Mark the continuation lines of every multi-line /* … */ block comment. - // Those lines are comment content, not code: they must be carried verbatim - // with the comment's opening line, never split off and reindented as if - // they were statements of their own. - inBlockComment := make([]bool, len(rawLines)) + // Mark the continuation lines of every multi-line token whose interior is + // not code: /* … */ block comments and every kind of string literal, + // including dollar-quoted ones (a dollar quote is just another way of + // quoting a string, whatever its tag, and its content may be any language). + // Those lines are token content, not statements: they must be carried + // verbatim with the token's opening line, never split off and reindented. + // On the line where such a token ends, tail holds the code that follows it + // so paren depth and statement terminators are still tracked. + inVerbatim := make([]bool, len(rawLines)) + tail := make([]string, len(rawLines)) + tailCol := make([]int, len(rawLines)) // column where tail[i] starts + cut := make([]int, len(rawLines)) // column where a multi-line token opens on this line, or -1 + for i := range cut { + cut[i] = -1 + } + lineStart := make([]int, len(rawLines)) + for i, off := 0, 0; i < len(rawLines); i++ { + lineStart[i] = off + off += len(rawLines[i]) + 1 + } + lineOf := func(off int) int { + return sort.Search(len(lineStart), func(i int) bool { return lineStart[i] > off }) - 1 + } for _, t := range lexer.Lex(normalised) { - if t.Kind != lexer.BlockComment { + switch t.Kind { + case lexer.BlockComment, lexer.String, lexer.EscapeString, lexer.BitString, lexer.DollarString: + default: continue } n := strings.Count(t.Text, "\n") - start := t.Line - 1 // lexer Line is 1-based within normalised - for k := 1; k <= n && start+k < len(inBlockComment); k++ { - inBlockComment[start+k] = true + if n == 0 || !literalTerminated(t) { + continue + } + start := lineOf(t.Off) + if cut[start] < 0 { + cut[start] = t.Off - lineStart[start] + } + for k := 1; k <= n && start+k < len(inVerbatim); k++ { + inVerbatim[start+k] = true + } + if start+n < len(tail) { + rest := normalised[t.Off+len(t.Text):] + if i := strings.IndexByte(rest, '\n'); i >= 0 { + rest = rest[:i] + } + tail[start+n] = rest + tailCol[start+n] = t.Off + len(t.Text) - lineStart[start+n] } } @@ -504,18 +552,105 @@ func formatBodyStatements(text string, st config.Style) string { stmt = nil } + scanLine := func(text string, openLit bool) { + var lastD0Kw string + for _, tok := range lexer.Lex(text) { + if tok.IsTrivia() || tok.Kind == lexer.EOF { + continue + } + switch tok.Kind { + case lexer.LParen, lexer.LBracket: + parenDepth++ + case lexer.RParen, lexer.RBracket: + if parenDepth > 0 { + parenDepth-- + } + case lexer.Semicolon: + if parenDepth == 0 { + // plpgsql_loop_collapse: fold empty FOR … LOOP END LOOP; to one line. + if st.PlpgsqlLoopCollapse && len(stmt) > 0 { + collapsed, ok := tryCollapseLoop(stmt, st) + if ok { + stmt = []bline{{text: collapsed, indent: ""}} + } + } + flush() + } + } + if parenDepth == 0 && tok.Kind == lexer.Ident { + lastD0Kw = lowerASCII(tok.Text) + switch lastD0Kw { + case "case": + caseDepth++ + case "end": + if caseDepth > 0 { + caseDepth-- + } + } + } else if parenDepth == 0 { + // A non-keyword token (string, operator, …) ends the run of + // keywords, so a line like `raise exception 'x'` does not end + // in the keyword EXCEPTION. + lastD0Kw = "" + } + } + + if openLit { + lastD0Kw = "" // the line ends inside a literal, not on a keyword + } + if parenDepth == 0 && len(stmt) > 0 { + switch lastD0Kw { + case "then": + // A THEN ending a CASE…WHEN branch is not a PL/pgSQL block + // opener; only one matching END closes the whole CASE, so + // treating each WHEN…THEN as a block open would permanently + // inflate blockDepth. + if caseDepth == 0 { + fw0 := lowerASCII(firstBodyKeyword(stmt[0].text)) + if fw0 != "elsif" && fw0 != "elseif" { + depthInc = true + } + flush() + } + case "loop", "begin": + fw0 := lowerASCII(firstBodyKeyword(stmt[0].text)) + if fw0 != "elsif" && fw0 != "elseif" { + depthInc = true + } + flush() + case "else", "exception": + if caseDepth > 0 { + break + } + flush() + } + } + } + for j, rawLine := range rawLines { line := strings.TrimRight(rawLine, "\r") - if inBlockComment[j] { - // Verbatim continuation of a multi-line block comment: glue it to the - // bline holding the comment's opening line. + if inVerbatim[j] { + // Verbatim continuation of a multi-line comment or literal: glue it + // to the bline holding the token's opening line. if len(stmt) > 0 { last := &stmt[len(stmt)-1] last.text += "\n" + line } else { stmt = append(stmt, bline{text: line}) } + if tail[j] != "" { + // A second literal may open later on this same line; scan only + // the code between the two. + tt, openLit := tail[j], false + if c := cut[j] - tailCol[j]; cut[j] >= 0 && c < len(tt) { + if c < 0 { + c = 0 + } + tt, openLit = tt[:c], true + } + scanLine(tt, openLit) + } continue } @@ -562,6 +697,12 @@ func formatBodyStatements(text string, st config.Style) string { } } + // Likewise never join onto a line that ends in a -- comment: the joined + // code would become part of the comment text. + if joinToPrev && endsInLineComment(stmt[len(stmt)-1].text) { + joinToPrev = false + } + if joinToPrev { last := &stmt[len(stmt)-1] last.text = strings.TrimRight(last.text, " \t") + " " + stripped @@ -569,70 +710,16 @@ func formatBodyStatements(text string, st config.Style) string { stmt = append(stmt, bline{text: stripped, indent: indent}) } - var lastD0Kw string - for _, tok := range lexer.Lex(stripped) { - if tok.IsTrivia() || tok.Kind == lexer.EOF { - continue - } - switch tok.Kind { - case lexer.LParen, lexer.LBracket: - parenDepth++ - case lexer.RParen, lexer.RBracket: - if parenDepth > 0 { - parenDepth-- - } - case lexer.Semicolon: - if parenDepth == 0 { - // plpgsql_loop_collapse: fold empty FOR … LOOP END LOOP; to one line. - if st.PlpgsqlLoopCollapse && len(stmt) > 0 { - collapsed, ok := tryCollapseLoop(stmt, st) - if ok { - stmt = []bline{{text: collapsed, indent: ""}} - } - } - flush() - } - } - if parenDepth == 0 && tok.Kind == lexer.Ident { - lastD0Kw = lowerASCII(tok.Text) - switch lastD0Kw { - case "case": - caseDepth++ - case "end": - if caseDepth > 0 { - caseDepth-- - } - } - } - } - - if parenDepth == 0 && len(stmt) > 0 { - switch lastD0Kw { - case "then": - // A THEN ending a CASE…WHEN branch is not a PL/pgSQL block - // opener; only one matching END closes the whole CASE, so - // treating each WHEN…THEN as a block open would permanently - // inflate blockDepth. - if caseDepth == 0 { - fw0 := lowerASCII(firstBodyKeyword(stmt[0].text)) - if fw0 != "elsif" && fw0 != "elseif" { - depthInc = true - } - flush() - } - case "loop", "begin": - fw0 := lowerASCII(firstBodyKeyword(stmt[0].text)) - if fw0 != "elsif" && fw0 != "elseif" { - depthInc = true - } - flush() - case "else", "exception": - if caseDepth > 0 { - break - } - flush() + // A multi-line literal that opens on this line is scanned only up to + // its opening quote; the rest is token content, not code. + scanText, openLit := stripped, false + if c := cut[j] - len(indent); cut[j] >= 0 && c < len(scanText) { + if c < 0 { + c = 0 } + scanText, openLit = scanText[:c], true } + scanLine(scanText, openLit) } flush() @@ -841,3 +928,91 @@ func leadingWhitespace(s string) string { } return s[:i] } + +// endsInLineComment reports whether s ends with a -- line comment. +func endsInLineComment(s string) bool { + toks := lexer.Lex(s) + for i := len(toks) - 1; i >= 0; i-- { + switch toks[i].Kind { + case lexer.EOF, lexer.Whitespace: + continue + } + return toks[i].Kind == lexer.LineComment + } + return false +} + +// literalTerminated reports whether a string or block-comment token is closed. +// Slices of nested dollar-quoted text can end mid-literal; an unterminated +// token there is a lexing artefact, not real literal content. +func literalTerminated(t lexer.Token) bool { + if t.Kind == lexer.BlockComment { + return strings.HasSuffix(t.Text, "*/") && len(t.Text) >= 4 + } + if t.Kind == lexer.DollarString { + _, _, _, ok := splitDollarQuote(t.Text) + return ok + } + return len(t.Text) >= 2 && strings.HasSuffix(t.Text, "'") +} + +// formatEmbeddedLiterals restyles every dollar-quoted literal in text whose +// content looks like code (see formatEmbedded). Everything else is untouched. +func formatEmbeddedLiterals(text string, st config.Style) string { + if !strings.Contains(text, "$") { + return text + } + var b strings.Builder + for _, t := range lexer.Lex(text) { + if t.Kind == lexer.DollarString { + b.WriteString(formatEmbedded(t.Text, st)) + } else { + b.WriteString(t.Text) + } + } + return b.String() +} + +// formatEmbedded formats a dollar-quoted literal as code when its content +// starts like code: DECLARE/BEGIN is formatted as a PL/pgSQL block and +// SELECT/INSERT/UPDATE/DELETE/WITH as DML. Any other content (SQL fragments, +// prose, other languages) is left exactly as written, as is the literal if the +// formatted result is not provably equivalent. +func formatEmbedded(lit string, st config.Style) string { + open, inner, close, ok := splitDollarQuote(lit) + if !ok { + return lit + } + // format()-style templates (%s, %I, %L, %1$s) are text, not SQL: restyling + // would split the placeholders, and the token-level safety gate can't see it. + if formatPlaceholder.MatchString(inner) { + return lit + } + var first string + for _, t := range lexer.Lex(inner) { + if t.IsTrivia() || t.Kind == lexer.EOF { + continue + } + if t.Kind == lexer.Ident { + first = lowerASCII(t.Text) + } + break + } + var res string + switch first { + case "declare", "begin": + res = formatBodyInner(inner, st) + case "select", "insert", "update", "delete", "with": + lead := inner[:len(inner)-len(strings.TrimLeft(inner, " \t\r\n"))] + trail := inner[len(strings.TrimRight(inner, " \t\r\n")):] + res = lead + strings.TrimSpace(File(parser.Parse(inner), st)) + trail + default: + return lit + } + if res == inner || !SemanticallyEqual(inner, res) || CommentsPreserved(inner, res) != nil { + return lit + } + return open + res + close +} + +var formatPlaceholder = regexp.MustCompile(`%(\d+\$)?-?\d*[sILlx]`) diff --git a/pkg/format/dml.go b/pkg/format/dml.go index 17aabc6..ae6d9e5 100644 --- a/pkg/format/dml.go +++ b/pkg/format/dml.go @@ -490,6 +490,12 @@ func dmlWrapSubquery(inner []cst.Tok, st config.Style) string { return b.String() } +// litNL stands in for a newline inside a multi-line string literal while DML +// text is assembled; formatDML's caller restores it (see restoreLiteralNewlines). +const litNL = "\x00\x01" + +func restoreLiteralNewlines(s string) string { return strings.ReplaceAll(s, litNL, "\n") } + // dmlInline renders toks on one line with keyword casing and proper spacing. // If toks[1:] contains comment trivia the function falls back to verbatimSpan // so no comment is lost. Subquery parens and CASE…END expressions embedded @@ -548,7 +554,17 @@ func dmlInline(toks []cst.Tok, st config.Style) string { prev = toks[i-1].Tok } nextIsLParen := i+1 < len(toks) && toks[i+1].Tok.Kind == lexer.LParen - b.WriteString(caseTextCtx(t.Tok, prev, nextIsLParen, st)) + txt := caseTextCtx(t.Tok, prev, nextIsLParen, st) + if t.Tok.Kind == lexer.DollarString { + txt = formatEmbedded(txt, st) + } + switch t.Tok.Kind { + case lexer.String, lexer.EscapeString, lexer.BitString, lexer.DollarString: + // Line breaks inside a literal are content: hide them from the + // line-based indent helpers that post-process this text. + txt = strings.ReplaceAll(txt, "\n", litNL) + } + b.WriteString(txt) i++ } return b.String() diff --git a/pkg/format/format.go b/pkg/format/format.go index da0ec2c..a2af80a 100644 --- a/pkg/format/format.go +++ b/pkg/format/format.go @@ -55,7 +55,7 @@ func (p *printer) writeItem(n cst.Node) { case *cst.Raw: switch { case isDMLStart(v.Toks): - p.b.WriteString(formatDML(v.Toks, p.st)) + p.b.WriteString(restoreLiteralNewlines(formatDML(v.Toks, p.st))) case isDoBlock(v.Toks): p.b.WriteString(formatDoBlock(v.Toks, p.st)) default: @@ -82,8 +82,10 @@ func isDoBlock(toks []cst.Tok) bool { // formatDoBlock formats a DO $$ ... $$ block by applying formatBody to the // dollar-quoted string and emitting DO + newline + formatted body. func formatDoBlock(toks []cst.Tok, st config.Style) string { - // Find the DO keyword, the dollar-string body, and the optional semicolon. + // Find the DO keyword, the dollar-string body, any other clause tokens + // (LANGUAGE x, before or after the body), and the optional semicolon. var doTok, bodyTok *cst.Tok + var pre, post []string hasSemi := false for i := range toks { t := &toks[i] @@ -101,6 +103,21 @@ func formatDoBlock(toks []cst.Tok, st config.Style) string { } if t.Tok.Kind == lexer.Semicolon { hasSemi = true + continue + } + // Clause tokens other than DO / body / ';' (e.g. LANGUAGE plpython3u). + // Any comment attached to one means we cannot safely relocate it. + if len(t.Comments()) > 0 { + return verbatimSpanFormatBody(toks, bodyTok, st) + } + txt := t.Tok.Text + if low == "language" { + txt = applyCase(txt, st.KeywordCase) + } + if bodyTok == nil { + pre = append(pre, txt) + } else { + post = append(post, txt) } } if doTok == nil || bodyTok == nil { @@ -110,8 +127,20 @@ func formatDoBlock(toks []cst.Tok, st config.Style) string { nl := st.Newline var b strings.Builder b.WriteString(applyCase(doTok.Tok.Text, st.KeywordCase)) + if len(pre) > 0 { + b.WriteString(" ") + b.WriteString(strings.Join(pre, " ")) + } b.WriteString(nl) - b.WriteString(formatBody(bodyTok.Tok.Text, st)) + if isPlpgsql(toks) { + b.WriteString(formatBody(bodyTok.Tok.Text, st)) + } else { + b.WriteString(bodyTok.Tok.Text) + } + if len(post) > 0 { + b.WriteString(nl) + b.WriteString(strings.Join(post, " ")) + } if hasSemi { b.WriteString(";") } @@ -193,7 +222,11 @@ func (p *printer) writeCreateFunction(cf *cst.CreateFunction) { } if cf.Body != nil { p.nl() - p.b.WriteString(formatBody(cf.Body.Tok.Text, p.st)) + if isPlpgsql(cst.Tokens(cf)) { + p.b.WriteString(formatBody(cf.Body.Tok.Text, p.st)) + } else { + p.b.WriteString(cf.Body.Tok.Text) + } } for _, clause := range cf.Tail { p.nl() @@ -445,7 +478,7 @@ func verbatimSpanFormatBody(toks []cst.Tok, bodyTok *cst.Tok, st config.Style) s b.WriteString(tr.Text) } } - if bodyTok != nil && t.Tok.Kind == lexer.DollarString && t.Tok.Off == bodyTok.Tok.Off { + if bodyTok != nil && t.Tok.Kind == lexer.DollarString && t.Tok.Off == bodyTok.Tok.Off && isPlpgsql(toks) { b.WriteString(formatBody(t.Tok.Text, st)) } else { b.WriteString(t.Tok.Text) @@ -546,3 +579,27 @@ func alignParamTypes(params []string) []string { } return out } + +// isPlpgsql reports whether the routine's body should be restyled. Bodies in +// any pl* language other than plpgsql (plpython3u, plperl, pltcl, ...), and in +// c or internal, are emitted verbatim; plpgsql, sql and everything else are +// formatted. A routine +// with no LANGUAGE clause is treated as sql and formatted. +func isPlpgsql(toks []cst.Tok) bool { + var sig []cst.Tok + for _, t := range toks { + if !t.Tok.IsTrivia() && t.Tok.Kind != lexer.EOF { + sig = append(sig, t) + } + } + for i := 0; i+1 < len(sig); i++ { + if sig[i].Is("language") { + name := strings.ToLower(strings.Trim(sig[i+1].Tok.Text, "'\"")) + if name == "c" || name == "internal" { + return false + } + return !strings.HasPrefix(name, "pl") || name == "plpgsql" + } + } + return true +} diff --git a/pkg/format/format_test.go b/pkg/format/format_test.go index 5aa089c..d48e1ee 100644 --- a/pkg/format/format_test.go +++ b/pkg/format/format_test.go @@ -200,3 +200,85 @@ func TestCorpusIdempotentAndSafe(t *testing.T) { func semanticallyEqual(a, b string) bool { return SemanticallyEqual(a, b) } + +func TestNonPlpgsqlBodyVerbatim(t *testing.T) { + body := "$$\ndeclare x = 1\nbegin = 2\n$$" + src := "create function f() returns void as " + body + " language plpython3u;" + if out := format(src); !strings.Contains(out, body) { + t.Fatalf("plpython3u body was modified:\n%s", out) + } +} + +func TestLanguageBodyGate(t *testing.T) { + body := "$$\ndeclare x int:=1;\nbegin\nnull;\nend\n$$" + for lang, verbatim := range map[string]bool{"plpgsql": false, "sql": false, "plpython3u": true, "plperl": true, "pltcl": true, "c": true, "internal": true} { + out := format("create function f() returns void as " + body + " language " + lang + ";") + if got := strings.Contains(out, body); got != verbatim { + t.Errorf("language %s: verbatim=%v, want %v\n%s", lang, got, verbatim, out) + } + } +} + +func TestDoBlockLanguageGate(t *testing.T) { + body := "$$\ndeclare x int:=1;\nbegin\nnull;\nend\n$$" + for src, verbatim := range map[string]bool{ + "do " + body + ";": false, + "do language plpgsql " + body + ";": false, + "do " + body + " language plpgsql;": false, + "do language plpython3u " + body + ";": true, + "do " + body + " language plperl;": true, + } { + if out := format(src); strings.Contains(out, body) != verbatim { + t.Errorf("%q: verbatim=%v, want %v\n%s", src, !verbatim, verbatim, out) + } + } +} + +func TestDoBlockKeepsLanguageClause(t *testing.T) { + for _, src := range []string{ + "do language plpython3u $$\nx = 1\n$$;", + "do $$\nx = 1\n$$ language plpython3u;", + } { + if out := format(src); !strings.Contains(strings.ToLower(out), "language plpython3u") { + t.Errorf("LANGUAGE clause lost:\n%s", out) + } + } +} + +func TestMultiLineStringLiteralVerbatim(t *testing.T) { + src := "create function f() returns void language plpgsql as $$\nDECLARE\n x int;\nBEGIN\n if x = 1 then\n raise exception E'A client\nTo resolve this', NEW.id;\n end if;\nEND\n$$;" + if out := format(src); !strings.Contains(out, "E'A client\nTo resolve this'") { + t.Errorf("multi-line string was reindented:\n%s", out) + } +} + +func TestCommentBeforeBeginKept(t *testing.T) { + src := "create function f() returns void language plpgsql as $$\nDECLARE\n x int = 1; --0=old, 1=new\nBEGIN\n null;\nEND\n$$;" + out := format(src) + if !strings.Contains(out, "--0=old, 1=new") { + t.Fatalf("comment before BEGIN lost:\n%s", out) + } + if again := format(out); again != out { + t.Fatalf("not idempotent:\n%s\n---\n%s", out, again) + } +} + +func TestEmbeddedDollarLiterals(t *testing.T) { + body := func(lit string) string { + return "create function f() returns void language plpgsql as $$\nDECLARE\n x text;\nBEGIN\n x = " + lit + ";\nEND\n$$;" + } + // format() template: must be untouched. + tmpl := "format($Q$select %s, %3$s from t$Q$, a)" + if out := format(body(tmpl)); !strings.Contains(out, "$Q$select %s, %3$s from t$Q$") { + t.Errorf("placeholder template modified:\n%s", out) + } + // SQL fragment (not a statement): untouched. + frag := "$s$ and (a=1) or b=2 $s$" + if out := format(body(frag)); !strings.Contains(out, frag) { + t.Errorf("fragment modified:\n%s", out) + } + // Statement: formatted as code. + if out := format(body("$q$select a,b from t where x=1$q$")); strings.Contains(out, "$q$select a,b from t where x=1$q$") { + t.Errorf("embedded select was not formatted:\n%s", out) + } +} diff --git a/testdata/corpus/test_mm_proc.pgsql b/testdata/corpus/test_mm_proc.pgsql index bc2cfb8..6b4724d 100644 --- a/testdata/corpus/test_mm_proc.pgsql +++ b/testdata/corpus/test_mm_proc.pgsql @@ -1238,62 +1238,71 @@ BEGIN ,format('%s', max(ifblnk(d.tblid,f.table_name)))::citext as subname ,($S $(select ('{'|| $S$ || string_agg(format($SS$'"%1$s": ' || json_build_object('value', json_agg(%2$s::text), - 'type', - '%3$s' - ) - : - : - text - $SS$, - c - . - mergetag, - c - . - field_name, - c - . - merge_type - ), - '|| '',''||' - ) - || - $S$ +'type', +'%3$s' +) +: +: +text +$SS$, +c +. +mergetag, +c +. +field_name, +c +. +merge_type +), +'|| '',''||' +) +|| +$S$ || '}' ) : : json - from $S$ || max ( - f - . - table_name - ) - || - ' ' - || - ifblnk ( - max - ( - f - . - filter_string - ), - 'where 1=1' - ) - || - ')' - ) - : - : - citext as qry , ( - $S - $(select ('{'|| $S$ || string_agg(format($SS$'"%1$s": ' || json_build_object('value', '[]', 'type','%3$s') - : - : - text - $SS$, + from + $S$ +|| +max +( +f +. +table_name +) +|| +' ' +|| +ifblnk +( +max +( +f +. +filter_string +), +'where 1=1' +) +|| +')' +) +: +: +citext +as +qry +, +( +$S +$(select ('{'|| $S$ || string_agg(format($SS$'"%1$s": ' || json_build_object('value', '[]', 'type','%3$s') +: +: +text +$SS$, c . mergetag, @@ -1308,12 +1317,24 @@ BEGIN ) || $S$ +|| +'}' +) +: +: +json +$S$ || - '}' + ')' ) : : - json $S$ || ')' ) : : citext as qryblnk , nv ( + citext + as + qryblnk + , + nv + ( max ( f @@ -1323,98 +1344,177 @@ BEGIN ) : : - citext as parent_order_string - from tmp_merge_init_src f + citext + as + parent_order_string + from + tmp_merge_init_src + f inner - join tmp_merge_init_src c - on c . rid_parent = f . rid and c . merge_type in ( + join + tmp_merge_init_src + c + on + c + . + rid_parent + = + f + . + rid + and + c + . + merge_type + in + ( G_MTYPE_TBLFIELD ) inner - join tmp_merge_init_fields d - on d . mergetag = c . mergetag and d . tblparent = r_lp_t . tblid - where f . grand_rid = r_lp_t . parent_rid and f . merge_type = G_MTYPE_TBLROOT + join + tmp_merge_init_fields + d + on + d + . + mergetag + = + c + . + mergetag + and + d + . + tblparent + = + r_lp_t + . + tblid + where + f + . + grand_rid + = + r_lp_t + . + parent_rid + and + f + . + merge_type + = + G_MTYPE_TBLROOT --and f.parent_rid = any(a_tblroot) --and d.table_level > 0 - group by f . rid ) loop - raise notice 'Inner Loop: %', r_lp_c . qry; - a_inner_selected = array_append(a_inner_selected, r_lp_t.table_name); + group + by + f + . + rid + ) + loop + raise + notice + 'Inner Loop: %', + r_lp_c + . + qry; + a_inner_selected + = array_append(a_inner_selected, r_lp_t.table_name); - m_execstr = format($S$%s|| '%s"%s":' || json_build_object('value',json_agg(%s::json %s)::json, 'type', '%s')::text %s$S$ + m_execstr + = format($S$%s|| '%s"%s":' || json_build_object('value',json_agg(%s::json %s)::json, 'type', '%s')::text %s$S$ ,m_execstr,',',r_lp_c.subname,r_lp_c.qry, r_lp_c.parent_order_string, r_lp_c.merge_type, E'\r\n'); - m_blankexec = format($S$%s|| '%s"%s":' || json_build_object('value',json_agg(%s::json %s)::json, 'type', '%s')::text %s$S$ + m_blankexec + = format($S$%s|| '%s"%s":' || json_build_object('value',json_agg(%s::json %s)::json, 'type', '%s')::text %s$S$ ,m_blankexec,',',r_lp_c.subname,r_lp_c.qryblnk, r_lp_c.parent_order_string, r_lp_c.merge_type, E'\r\n'); - end loop; + end loop; - if ifblnk(r_lp_t.parent_table_name,'') = '' - then - m_execstr = format(E'select (''{'' %s \r\n || ''}'')::json ;',m_execstr ); - else - select string_agg(s.filter_string, ' ') - from tmp_merge_init_fields f + if + ifblnk(r_lp_t.parent_table_name,'') = '' + then + m_execstr = format(E'select (''{'' %s \r\n || ''}'')::json ;',m_execstr ); + else + select string_agg(s.filter_string, ' ') + from tmp_merge_init_fields f inner join tmp_merge_init_src s on s.mergetag = f.mergetag and s.merge_type = G_MTYPE_FILTER - where f.source = r_lp_t.source into m_tablefilter ; + where f.source = r_lp_t.source into m_tablefilter + ; - m_execstr = format(E'select (''{'' %s \r\n || ''}'')::json \r\nfrom %s \r\n%s;' + m_execstr + = format(E'select (''{'' %s \r\n || ''}'')::json \r\nfrom %s \r\n%s;' ,m_execstr, r_lp_t.parent_table_name,ifblnk(r_lp_t.parent_filter_string, ' where 1=1 ') || nv(m_execfilter) || nv(m_tablefilter) ); - end if; + end if; - m_blankexec = format(E'select (''{'' %s \r\n || ''}'')::json \r\n;',m_blankexec ); + m_blankexec + = format(E'select (''{'' %s \r\n || ''}'')::json \r\n;',m_blankexec ); - select r.p_retval, r.p_errmsg, r.p_json - > 'str' - from exec_json(m_execstr, 'str json') r into m_retval,m_errmsg, m_json; + select r.p_retval, r.p_errmsg, r.p_json - > 'str' + from exec_json(m_execstr, 'str json') r into m_retval,m_errmsg, m_json; - if m_json is null - then + if + m_json is null + then - select r.p_retval, r.p_errmsg, r.p_json - > 'str' - from exec_json(m_execstr, 'str json') r into m_retval,m_errmsg, m_json; - end if; + select r.p_retval, r.p_errmsg, r.p_json - > 'str' + from exec_json(m_execstr, 'str json') r into m_retval,m_errmsg, m_json; + end if; - m_debug_exestr = nv(m_debug_exestr) || E'\r\n/*'|| nv(r_lp_t.parent_table_name) || ' len:' || nv(length(m_json::text)) ||E'*/ \r\n' || nv(m_execstr) || E'\r\n '; + m_debug_exestr + = nv(m_debug_exestr) || E'\r\n/*'|| nv(r_lp_t.parent_table_name) || ' len:' || nv(length(m_json::text)) ||E'*/ \r\n' || nv(m_execstr) || E'\r\n '; - if m_json_full_complex is null - then - m_json_full_complex = jsonb_build_object(r_lp_t.tblid::text,m_json); - end if; + if + m_json_full_complex is null + then + m_json_full_complex = jsonb_build_object(r_lp_t.tblid::text,m_json); + end if; - if (m_json_full_complex->r_lp_t.tblid::text) is null - then - m_json_full_complex = jsonb_set(m_json_full_complex, format('{%s}',r_lp_t.tblid)::text[], m_json::jsonb,true); - else - m_json_full_complex = jsonb_set(m_json_full_complex, format('{%s}',r_lp_t.tblid)::text[], _jsonb_object_cat(m_json_full_complex->r_lp_t.tblid,m_json::jsonb),true); - end if; + if + (m_json_full_complex->r_lp_t.tblid::text) is null + then + m_json_full_complex = jsonb_set(m_json_full_complex, format('{%s}',r_lp_t.tblid)::text[], m_json::jsonb,true); + else + m_json_full_complex = jsonb_set(m_json_full_complex, format('{%s}',r_lp_t.tblid)::text[], _jsonb_object_cat(m_json_full_complex->r_lp_t.tblid,m_json::jsonb),true); + end if; - -- perform log_event(m_funcname,format('Complex Tables %s=%s m_json_full_complex=%s' ,p_data_prefix,p_data_rid + -- perform log_event(m_funcname,format('Complex Tables %s=%s m_json_full_complex=%s' ,p_data_prefix,p_data_rid -- ,m_json_full_complex::text -- ),bt_enum('eventlog','local notice') -- --,(select jsonb_agg(row_to_json(f)::jsonb) from tmp_merge_init_fields f)::text -- ); - m_execfilter = ''; - m_execstr = ''; - m_blankexec = ''; - m_exec_orderstr = ''; - m_comma = ''; - m_tablefilter = ''; + m_execfilter + = ''; + m_execstr + = ''; + m_blankexec + = ''; + m_exec_orderstr + = ''; + m_comma + = ''; + m_tablefilter + = ''; end if; - if nv(m_comma) = '' and length(m_execstr) > 2 + if + nv(m_comma) = '' and length(m_execstr) > 2 then - m_comma = ','; + m_comma = ','; end if; - end loop; + end loop; - if G_DEBUG - then - perform pl_writefile(r_template.debugsql_filename, convert_to(m_debug_exestr,'utf8')); - end if; + if + G_DEBUG + then + perform pl_writefile(r_template.debugsql_filename, convert_to(m_debug_exestr,'utf8')); + end if; -EXCEPTION + EXCEPTION WHEN others THEN GET STACKED DIAGNOSTICS m_errmsg = MESSAGE_TEXT @@ -1423,51 +1523,67 @@ EXCEPTION ,m_errhint = PG_EXCEPTION_HINT ,m_errstate = RETURNED_SQLSTATE; - m_errmsg = format(E'Merge failed to complete. Merge fields are not setup correctly. \r\nPlease check the template. \r\nThere could be table merge tags outside of a table. \r\nDetail Error: \r\n%s',m_errmsg); - m_errmsg = nv(m_errmsg) || format(E'\r\nExecString: %s ', ifblnk(m_execstr,m_debug_exestr)); - m_errmsg = nv(m_errmsg) || format(E'\r\nError Detail: %s , %s, %s, %s', m_errdetail,m_errcontext,m_errhint,m_errstate); + m_errmsg += format(E'Merge failed to complete. Merge fields are not setup correctly. \r\nPlease check the template. \r\nThere could be table merge tags outside of a table. \r\nDetail Error: \r\n%s',m_errmsg); + m_errmsg += nv(m_errmsg) || format(E'\r\nExecString: %s ', ifblnk(m_execstr,m_debug_exestr)); + m_errmsg += nv(m_errmsg) || format(E'\r\nError Detail: %s , %s, %s, %s', m_errdetail,m_errcontext,m_errhint,m_errstate); - if G_DEBUG + if +G_DEBUG then m_errmsg = format(E'%s \r\nDebug file: %s',m_errmsg, r_template.debugsql_filename); - perform pl_writefile(r_template.debugsql_filename, convert_to(m_debug_exestr,'utf8')); + perform +pl_writefile(r_template.debugsql_filename, convert_to(m_debug_exestr,'utf8')); end if; - p_retval = 1; - p_errmsg = m_errmsg; + p_retval += 1; + p_errmsg += m_errmsg; - m_json_full_complex = _jsonb_object_cat(m_json_full_complex, jsonb_build_object('p_retval',p_retval,'p_errmsg',p_errmsg)); + m_json_full_complex += _jsonb_object_cat(m_json_full_complex, jsonb_build_object('p_retval',p_retval,'p_errmsg',p_errmsg)); return; --raise exception '%', m_errmsg using hint = 'in merge jsonbuild process'; END; -------------------------------------------------------------------------------------------------------- - m_json_full = json_build_object('fields',m_json_full, 'complexfields',m_json_full_complex); + m_json_full += json_build_object('fields',m_json_full, 'complexfields',m_json_full_complex); - if G_DEBUG + if +G_DEBUG then perform pl_writefile(r_template.debug_filename, convert_to(m_json_full::text,'utf8')); end if; - if G_BENCHMARK = 1 + if +G_BENCHMARK = 1 then perform log_event(m_funcname,format('Perf Complex Fields 2SinceStart: %s Duration: %s', clock_timestamp() - m_start, clock_timestamp() - m_ltime),bt_enum('eventlog','local notice')); - m_ltime = clock_timestamp(); + m_ltime += clock_timestamp(); end if; - if m_returnvalues + if +m_returnvalues then p_doc = convert_to(m_json_full::text, 'utf8'); - p_docguid = 'json:see->p_doc'; + p_docguid += 'json:see->p_doc'; else if G_BENCHMARK = 1 then perform log_event(m_funcname,format('Perf Before pl_mailmerge SinceStart: %s Duration: %s', clock_timestamp() - m_start, clock_timestamp() - m_ltime),bt_enum('eventlog','local notice')); - m_ltime = clock_timestamp(); + m_ltime += clock_timestamp(); end if; - if m_hasfilestream + if +m_hasfilestream then --filesystem select r.p_retval @@ -1476,33 +1592,41 @@ select r.p_retval from pl_mailmerge(format('merge_%s', p_doctype), r_template.filepath, r_doc.filepath, m_json_full::text, 1 /*New mode, new tags*/) r into r_retval; -if G_BENCHMARK = 1 +if +G_BENCHMARK = 1 then perform log_event(m_funcname,format('Perf After pl_mailmerge SinceStart: %s Duration: %s', clock_timestamp() - m_start, clock_timestamp() - m_ltime),bt_enum('eventlog','local notice')); - m_ltime = clock_timestamp(); + m_ltime += clock_timestamp(); end if; - if r_retval.p_retval = 1 + if +r_retval.p_retval = 1 then raise '%',r_retval.p_errmsg; - elseif r_retval.p_retval = 2 + elseif +r_retval.p_retval = 2 then p_retval = 2; - p_errmsg = r_retval.p_errmsg; + p_errmsg += r_retval.p_errmsg; end if; select r.p_retval, r.p_errmsg from f_tempfile_add(r_doc.filepath, p_doctype, r_doc.guid, 600, m_data_rid, m_data_prefix) r into m_retval, m_errmsg; -p_docguid = r_doc.guid; +p_docguid += r_doc.guid; select r.p_outfile, r.p_retval, r.p_errmsg from pl_readfile(r_doc.filepath) r into p_doc, m_retval, m_errmsg; -if m_retval = 0 +if +m_retval = 0 then perform pl_deletefile(r_doc.filepath); - perform pl_deletefile(r_template.filepath); + perform +pl_deletefile(r_template.filepath); end if; else @@ -1522,32 +1646,41 @@ select r.p_retval from pl_mailmerge(format('merge_%s', p_doctype), null, null, m_json_full::text , 1 /*New mode, new tags*/, r_template.blob) r into r_retval; -if G_BENCHMARK = 1 +if +G_BENCHMARK = 1 then perform log_event(m_funcname,format('Perf After pl_mailmerge Stream SinceStart: %s Duration: %s', clock_timestamp() - m_start, clock_timestamp() - m_ltime),bt_enum('eventlog','local notice')); - m_ltime = clock_timestamp(); + m_ltime += clock_timestamp(); end if; - r_doc.blob = r_retval.p_file; - p_doc = r_retval.p_file; + r_doc.blob += r_retval.p_file; + p_doc += r_retval.p_file; - if r_retval.p_retval = 1 + if +r_retval.p_retval = 1 then raise '%',r_retval.p_errmsg; - elseif r_retval.p_retval = 2 + elseif +r_retval.p_retval = 2 then p_retval = 2; - p_errmsg = r_retval.p_errmsg; + p_errmsg += r_retval.p_errmsg; end if; end if; end if; - if G_BENCHMARK in (1,2) + if +G_BENCHMARK in (1,2) then perform log_event(m_funcname,format('Perf Merge End (%s,%s) SinceStart: %s Duration: %s',p_doctype,p_data_rid, clock_timestamp() - m_start, clock_timestamp() - m_ltime),bt_enum('eventlog','local notice')); - m_ltime = clock_timestamp(); + m_ltime += clock_timestamp(); end if; EXCEPTION @@ -1559,16 +1692,21 @@ WHEN others THEN ,m_errhint = PG_EXCEPTION_HINT ,m_errstate = RETURNED_SQLSTATE; - p_errmsg := get_err_msg(m_funcname, m_errmsg, m_errcontext, m_errdetail, m_errhint, m_errstate); - p_retval = 1; + p_errmsg +:= get_err_msg(m_funcname, m_errmsg, m_errcontext, m_errdetail, m_errhint, m_errstate); + p_retval += 1; - p_errmsg = nv(p_errmsg) || nv(format(E'\r\n p_doctype:%s, p_commtype:%s, p_data_prefix:%s, p_data_rid:%s, p_filterdata:%s' + p_errmsg += nv(p_errmsg) || nv(format(E'\r\n p_doctype:%s, p_commtype:%s, p_data_prefix:%s, p_data_rid:%s, p_filterdata:%s' ,p_doctype,p_commtype,p_data_prefix,p_data_rid, p_filterdata)); - if G_DEBUG + if +G_DEBUG then perform pl_writefile(r_template.debugsql_filename, convert_to(m_debug_exestr,'utf8')); - perform pl_writefile(r_template.debug_filename, convert_to(m_json_full::text,'utf8')); + perform +pl_writefile(r_template.debug_filename, convert_to(m_json_full::text,'utf8')); end if; END;