feat: initial plan
This commit is contained in:
Hein
@@ -0,0 +1,55 @@
|
||||
// Package config defines PgTidy's formatter (and, later, linter) configuration.
|
||||
//
|
||||
// Defaults encode the project house style reverse-engineered from the corpus.
|
||||
// A future change will load/merge these from a discovered .pgtidy.yaml file.
|
||||
package config
|
||||
|
||||
// Case controls keyword/identifier casing.
|
||||
type Case string
|
||||
|
||||
const (
|
||||
CaseUpper Case = "upper"
|
||||
CaseLower Case = "lower"
|
||||
// CasePreserve leaves the token text unchanged.
|
||||
CasePreserve Case = "preserve"
|
||||
)
|
||||
|
||||
// CommaStyle controls where separators sit in multi-line lists.
|
||||
type CommaStyle string
|
||||
|
||||
const (
|
||||
// CommaLeading puts the comma at the start of the continuation line
|
||||
// (",col"), the house style.
|
||||
CommaLeading CommaStyle = "leading"
|
||||
// CommaTrailing puts the comma at the end of the preceding line ("col,").
|
||||
CommaTrailing CommaStyle = "trailing"
|
||||
)
|
||||
|
||||
// Style is the formatter configuration.
|
||||
type Style struct {
|
||||
// Indent is one indentation level (default two spaces).
|
||||
Indent string
|
||||
// Newline is the line terminator emitted by the formatter.
|
||||
Newline string
|
||||
// KeywordCase controls SQL keyword casing (types excluded — see TypeCase).
|
||||
KeywordCase Case
|
||||
// IdentCase controls unquoted identifier casing (quoted identifiers are
|
||||
// never touched).
|
||||
IdentCase Case
|
||||
// TypeCase controls built-in type-name casing.
|
||||
TypeCase Case
|
||||
// Commas controls list separator placement.
|
||||
Commas CommaStyle
|
||||
}
|
||||
|
||||
// Default returns the house-style configuration.
|
||||
func Default() Style {
|
||||
return Style{
|
||||
Indent: " ",
|
||||
Newline: "\n",
|
||||
KeywordCase: CaseUpper,
|
||||
IdentCase: CaseLower,
|
||||
TypeCase: CaseLower,
|
||||
Commas: CommaLeading,
|
||||
}
|
||||
}
|
||||
+173
@@ -0,0 +1,173 @@
|
||||
// Package cst defines PgTidy's concrete syntax tree.
|
||||
//
|
||||
// The CST is lossless: every lexer token (significant tokens and the trivia —
|
||||
// whitespace/comments — attached to them) is reachable, so File.Source()
|
||||
// reproduces the original input byte-for-byte. The parser builds structured
|
||||
// nodes only where it is confident; everything else is captured verbatim in a
|
||||
// Raw node. This makes "graceful degradation" a property of the data model
|
||||
// rather than something the printer must remember to do.
|
||||
package cst
|
||||
|
||||
import (
|
||||
"strings"
|
||||
|
||||
"github.com/hein/pgtidy/pkg/lexer"
|
||||
)
|
||||
|
||||
// Trivia is a run of whitespace/comment tokens preceding a significant token.
|
||||
type Trivia []lexer.Token
|
||||
|
||||
// Tok is a significant (non-trivia) token together with the trivia that
|
||||
// immediately precedes it in the source.
|
||||
type Tok struct {
|
||||
Lead Trivia
|
||||
Tok lexer.Token
|
||||
}
|
||||
|
||||
// Text returns the significant token's text.
|
||||
func (t Tok) Text() string { return t.Tok.Text }
|
||||
|
||||
// Is reports whether the token is an unquoted word equal (case-insensitively)
|
||||
// to kw.
|
||||
func (t Tok) Is(kw string) bool {
|
||||
return t.Tok.Kind == lexer.Ident && strings.EqualFold(t.Tok.Text, kw)
|
||||
}
|
||||
|
||||
// Comments returns just the comment tokens from the leading trivia.
|
||||
func (t Tok) Comments() []lexer.Token {
|
||||
var out []lexer.Token
|
||||
for _, tr := range t.Lead {
|
||||
if tr.Kind == lexer.LineComment || tr.Kind == lexer.BlockComment {
|
||||
out = append(out, tr)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// Attach converts a lossless lexer stream into significant tokens, each
|
||||
// carrying its leading trivia, plus any trailing trivia before EOF.
|
||||
func Attach(toks []lexer.Token) (sig []Tok, trailing Trivia) {
|
||||
var lead Trivia
|
||||
for _, t := range toks {
|
||||
if t.Kind == lexer.EOF {
|
||||
trailing = lead
|
||||
break
|
||||
}
|
||||
if t.IsTrivia() {
|
||||
lead = append(lead, t)
|
||||
continue
|
||||
}
|
||||
sig = append(sig, Tok{Lead: lead, Tok: t})
|
||||
lead = nil
|
||||
}
|
||||
return sig, trailing
|
||||
}
|
||||
|
||||
// Node is any element of a File.
|
||||
type Node interface {
|
||||
appendTokens(*[]Tok)
|
||||
}
|
||||
|
||||
// Tokens returns all significant tokens of a node in source order.
|
||||
func Tokens(n Node) []Tok {
|
||||
var t []Tok
|
||||
n.appendTokens(&t)
|
||||
return t
|
||||
}
|
||||
|
||||
// File is the whole parsed input.
|
||||
type File struct {
|
||||
Items []Node
|
||||
Trailing Trivia // trivia after the last significant token
|
||||
}
|
||||
|
||||
// Source reconstructs the original input from the tree. With a correct parse
|
||||
// this is byte-for-byte identical to the lexer input.
|
||||
func (f *File) Source() string {
|
||||
var toks []Tok
|
||||
for _, it := range f.Items {
|
||||
it.appendTokens(&toks)
|
||||
}
|
||||
var b strings.Builder
|
||||
for _, t := range toks {
|
||||
for _, tr := range t.Lead {
|
||||
b.WriteString(tr.Text)
|
||||
}
|
||||
b.WriteString(t.Tok.Text)
|
||||
}
|
||||
for _, tr := range f.Trailing {
|
||||
b.WriteString(tr.Text)
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
|
||||
// Raw is an unstructured statement: a verbatim run of significant tokens. This
|
||||
// is the graceful-degradation fallback for anything the parser does not (yet)
|
||||
// structure.
|
||||
type Raw struct {
|
||||
Toks []Tok
|
||||
}
|
||||
|
||||
func (r *Raw) appendTokens(out *[]Tok) { *out = append(*out, r.Toks...) }
|
||||
|
||||
// Param is one entry in a function/procedure parameter list, with the comma
|
||||
// that separates it from the next entry (nil on the last parameter).
|
||||
type Param struct {
|
||||
Toks []Tok
|
||||
Sep *Tok // trailing comma, nil if last
|
||||
}
|
||||
|
||||
// CreateFunction is a parsed CREATE [OR REPLACE] FUNCTION|PROCEDURE statement.
|
||||
//
|
||||
// Field order matches source order; appendTokens emits them contiguously so the
|
||||
// node round-trips exactly.
|
||||
type CreateFunction struct {
|
||||
Head []Tok // CREATE [OR REPLACE] FUNCTION|PROCEDURE
|
||||
Name []Tok // name (possibly schema-qualified)
|
||||
LParen Tok //
|
||||
Params []Param // parameter list (may be empty)
|
||||
RParen Tok //
|
||||
Options [][]Tok // option clauses before the body (RETURNS/LANGUAGE/VOLATILE/...)
|
||||
As *Tok // the AS keyword, if present
|
||||
Body *Tok // the body token (dollar-quoted string or string), if present
|
||||
Tail [][]Tok // option clauses after the body (rare)
|
||||
Semi *Tok // terminating semicolon, if present
|
||||
}
|
||||
|
||||
// IsProcedure reports whether this is a PROCEDURE (vs FUNCTION).
|
||||
func (c *CreateFunction) IsProcedure() bool {
|
||||
for _, t := range c.Head {
|
||||
if t.Is("procedure") {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func (c *CreateFunction) appendTokens(out *[]Tok) {
|
||||
*out = append(*out, c.Head...)
|
||||
*out = append(*out, c.Name...)
|
||||
*out = append(*out, c.LParen)
|
||||
for _, p := range c.Params {
|
||||
*out = append(*out, p.Toks...)
|
||||
if p.Sep != nil {
|
||||
*out = append(*out, *p.Sep)
|
||||
}
|
||||
}
|
||||
*out = append(*out, c.RParen)
|
||||
for _, cl := range c.Options {
|
||||
*out = append(*out, cl...)
|
||||
}
|
||||
if c.As != nil {
|
||||
*out = append(*out, *c.As)
|
||||
}
|
||||
if c.Body != nil {
|
||||
*out = append(*out, *c.Body)
|
||||
}
|
||||
for _, cl := range c.Tail {
|
||||
*out = append(*out, cl...)
|
||||
}
|
||||
if c.Semi != nil {
|
||||
*out = append(*out, *c.Semi)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,310 @@
|
||||
// Package format renders a cst.File back to source text in the configured
|
||||
// house style.
|
||||
//
|
||||
// Scope (V1): CREATE FUNCTION/PROCEDURE headers are laid out to house style
|
||||
// (one parameter per line with leading commas, each option clause on its own
|
||||
// line, AS/$$ on their own lines). The PL/pgSQL body and any statement the
|
||||
// parser left as cst.Raw are emitted verbatim — this upholds the safety
|
||||
// invariants while body formatting is built out (task #4).
|
||||
//
|
||||
// Guarantees: formatting changes only trivia/layout (never literal/identifier
|
||||
// semantics), and is idempotent.
|
||||
package format
|
||||
|
||||
import (
|
||||
"strings"
|
||||
|
||||
"github.com/hein/pgtidy/pkg/config"
|
||||
"github.com/hein/pgtidy/pkg/cst"
|
||||
"github.com/hein/pgtidy/pkg/lexer"
|
||||
)
|
||||
|
||||
// File formats a parsed file with the given style.
|
||||
func File(f *cst.File, st config.Style) string {
|
||||
p := &printer{st: st}
|
||||
for i, item := range f.Items {
|
||||
toks := cst.Tokens(item)
|
||||
if len(toks) == 0 {
|
||||
continue
|
||||
}
|
||||
lead := toks[0].Lead
|
||||
if i > 0 {
|
||||
p.nl()
|
||||
if hasBlankLine(lead) {
|
||||
p.nl()
|
||||
}
|
||||
}
|
||||
p.leadingComments(lead)
|
||||
p.writeItem(item)
|
||||
}
|
||||
p.trailingComments(f.Trailing)
|
||||
return ensureTrailingNewline(p.b.String(), st.Newline)
|
||||
}
|
||||
|
||||
type printer struct {
|
||||
st config.Style
|
||||
b strings.Builder
|
||||
}
|
||||
|
||||
func (p *printer) nl() { p.b.WriteString(p.st.Newline) }
|
||||
|
||||
func (p *printer) writeItem(n cst.Node) {
|
||||
switch v := n.(type) {
|
||||
case *cst.CreateFunction:
|
||||
p.writeCreateFunction(v)
|
||||
default:
|
||||
p.b.WriteString(verbatimSpan(cst.Tokens(n)))
|
||||
}
|
||||
}
|
||||
|
||||
func (p *printer) writeCreateFunction(cf *cst.CreateFunction) {
|
||||
// Safety: if the header carries comments we cannot confidently relocate,
|
||||
// emit the whole statement verbatim rather than risk dropping them.
|
||||
if headerHasComments(cf) {
|
||||
p.b.WriteString(verbatimSpan(cst.Tokens(cf)))
|
||||
return
|
||||
}
|
||||
|
||||
head := p.inline(cf.Head)
|
||||
name := p.inline(cf.Name)
|
||||
p.b.WriteString(head)
|
||||
if name != "" {
|
||||
p.b.WriteString(" ")
|
||||
p.b.WriteString(name)
|
||||
}
|
||||
p.b.WriteString("(")
|
||||
|
||||
first := p.st.Indent + " " // align item text one column past the comma
|
||||
cont := p.st.Indent
|
||||
for i, param := range cf.Params {
|
||||
p.nl()
|
||||
text := p.inline(param.Toks)
|
||||
if i == 0 || p.st.Commas != config.CommaLeading {
|
||||
p.b.WriteString(first)
|
||||
p.b.WriteString(text)
|
||||
if p.st.Commas == config.CommaTrailing && i < len(cf.Params)-1 {
|
||||
p.b.WriteString(",")
|
||||
}
|
||||
} else {
|
||||
p.b.WriteString(cont)
|
||||
p.b.WriteString(",")
|
||||
p.b.WriteString(text)
|
||||
}
|
||||
}
|
||||
p.nl()
|
||||
p.b.WriteString(")")
|
||||
|
||||
for _, clause := range cf.Options {
|
||||
p.nl()
|
||||
p.b.WriteString(p.inline(clause))
|
||||
}
|
||||
if cf.As != nil {
|
||||
p.nl()
|
||||
p.b.WriteString(p.inline([]cst.Tok{{Tok: cf.As.Tok}}))
|
||||
}
|
||||
if cf.Body != nil {
|
||||
p.nl()
|
||||
p.b.WriteString(cf.Body.Tok.Text) // body emitted verbatim (formatted later)
|
||||
}
|
||||
for _, clause := range cf.Tail {
|
||||
p.nl()
|
||||
p.b.WriteString(p.inline(clause))
|
||||
}
|
||||
if cf.Semi != nil {
|
||||
p.b.WriteString(";")
|
||||
}
|
||||
}
|
||||
|
||||
// inline renders a run of tokens on one line, applying spacing and casing.
|
||||
// If the run contains comment trivia it is emitted verbatim to avoid losing
|
||||
// or misplacing the comments.
|
||||
func (p *printer) inline(toks []cst.Tok) string {
|
||||
if len(toks) == 0 {
|
||||
return ""
|
||||
}
|
||||
// A comment on the first token's lead is the span's leading/separation
|
||||
// comment, handled by the caller and never emitted here, so it does not
|
||||
// force verbatim. Internal comments (on later tokens) do.
|
||||
if anyComment(toks[1:]) {
|
||||
return verbatimSpan(toks)
|
||||
}
|
||||
var b strings.Builder
|
||||
for i, t := range toks {
|
||||
if i > 0 && needSpace(toks[i-1].Tok, t.Tok) {
|
||||
b.WriteByte(' ')
|
||||
}
|
||||
b.WriteString(caseText(t.Tok, p.st))
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
|
||||
func (p *printer) leadingComments(lead cst.Trivia) {
|
||||
for _, tr := range lead {
|
||||
if tr.Kind == lexer.LineComment || tr.Kind == lexer.BlockComment {
|
||||
p.b.WriteString(strings.TrimRight(tr.Text, " \t"))
|
||||
p.nl()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (p *printer) trailingComments(lead cst.Trivia) {
|
||||
cs := commentsOf(lead)
|
||||
for _, c := range cs {
|
||||
p.nl()
|
||||
p.b.WriteString(strings.TrimRight(c.Text, " \t"))
|
||||
}
|
||||
}
|
||||
|
||||
// --- spacing & casing ---
|
||||
|
||||
// tightOps are operators printed without surrounding spaces.
|
||||
var tightOps = map[string]bool{"::": true, ":": true, "->": true, "->>": true}
|
||||
|
||||
func needSpace(a, b lexer.Token) bool {
|
||||
// No space after.
|
||||
switch a.Kind {
|
||||
case lexer.LParen, lexer.LBracket, lexer.Dot:
|
||||
return false
|
||||
case lexer.Operator:
|
||||
if tightOps[a.Text] {
|
||||
return false
|
||||
}
|
||||
}
|
||||
// No space before.
|
||||
switch b.Kind {
|
||||
case lexer.RParen, lexer.RBracket, lexer.Comma, lexer.Semicolon, lexer.Dot:
|
||||
return false
|
||||
case lexer.LParen:
|
||||
switch a.Kind {
|
||||
case lexer.Ident, lexer.QuotedIdent, lexer.RParen, lexer.RBracket, lexer.Param:
|
||||
return false // function call / type modifier
|
||||
}
|
||||
case lexer.Operator:
|
||||
if tightOps[b.Text] {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func caseText(t lexer.Token, st config.Style) string {
|
||||
if t.Kind != lexer.Ident {
|
||||
return t.Text // only unquoted words are re-cased
|
||||
}
|
||||
low := lowerASCII(t.Text)
|
||||
switch {
|
||||
case isTypeName(low):
|
||||
return applyCase(t.Text, st.TypeCase)
|
||||
case isKeyword(low):
|
||||
return applyCase(t.Text, st.KeywordCase)
|
||||
default:
|
||||
return applyCase(t.Text, st.IdentCase)
|
||||
}
|
||||
}
|
||||
|
||||
func applyCase(s string, c config.Case) string {
|
||||
switch c {
|
||||
case config.CaseUpper:
|
||||
return strings.ToUpper(s)
|
||||
case config.CaseLower:
|
||||
return strings.ToLower(s)
|
||||
default:
|
||||
return s
|
||||
}
|
||||
}
|
||||
|
||||
// --- helpers ---
|
||||
|
||||
// headerHasComments reports whether the function header carries comment trivia
|
||||
// the formatter cannot confidently relocate. The first token's leading trivia
|
||||
// is excluded: that is the statement's leading comment, which File() emits
|
||||
// separately. The body token's own text is excluded too (it is emitted
|
||||
// verbatim), but a comment in front of the body is caught.
|
||||
func headerHasComments(cf *cst.CreateFunction) bool {
|
||||
all := cst.Tokens(cf)
|
||||
for i, t := range all {
|
||||
if i == 0 {
|
||||
continue
|
||||
}
|
||||
if t.Tok.Kind == lexer.Semicolon {
|
||||
continue
|
||||
}
|
||||
if hasComment(t) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func anyComment(toks []cst.Tok) bool {
|
||||
for _, t := range toks {
|
||||
if hasComment(t) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func hasComment(t cst.Tok) bool {
|
||||
for _, tr := range t.Lead {
|
||||
if tr.Kind == lexer.LineComment || tr.Kind == lexer.BlockComment {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func commentsOf(lead cst.Trivia) []lexer.Token {
|
||||
var out []lexer.Token
|
||||
for _, tr := range lead {
|
||||
if tr.Kind == lexer.LineComment || tr.Kind == lexer.BlockComment {
|
||||
out = append(out, tr)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// verbatimSpan emits tokens exactly as in source, excluding the leading trivia
|
||||
// of the first token (separation is controlled by the caller).
|
||||
func verbatimSpan(toks []cst.Tok) string {
|
||||
var b strings.Builder
|
||||
for i, t := range toks {
|
||||
if i > 0 {
|
||||
for _, tr := range t.Lead {
|
||||
b.WriteString(tr.Text)
|
||||
}
|
||||
}
|
||||
b.WriteString(t.Tok.Text)
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
|
||||
// hasBlankLine reports whether leading whitespace trivia contains a blank line
|
||||
// (two or more newlines), indicating the author wanted statements separated.
|
||||
func hasBlankLine(lead cst.Trivia) bool {
|
||||
n := 0
|
||||
for _, tr := range lead {
|
||||
if tr.Kind == lexer.Whitespace {
|
||||
n += strings.Count(tr.Text, "\n")
|
||||
}
|
||||
}
|
||||
return n >= 2
|
||||
}
|
||||
|
||||
func ensureTrailingNewline(s, nl string) string {
|
||||
s = strings.TrimRight(s, "\n")
|
||||
if s == "" {
|
||||
return s
|
||||
}
|
||||
return s + nl
|
||||
}
|
||||
|
||||
func lowerASCII(s string) string {
|
||||
b := []byte(s)
|
||||
for i, c := range b {
|
||||
if c >= 'A' && c <= 'Z' {
|
||||
b[i] = c + 32
|
||||
}
|
||||
}
|
||||
return string(b)
|
||||
}
|
||||
@@ -0,0 +1,118 @@
|
||||
package format
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/hein/pgtidy/pkg/config"
|
||||
"github.com/hein/pgtidy/pkg/lexer"
|
||||
"github.com/hein/pgtidy/pkg/parser"
|
||||
)
|
||||
|
||||
func format(src string) string {
|
||||
return File(parser.Parse(src), config.Default())
|
||||
}
|
||||
|
||||
func TestFormatHeaderGolden(t *testing.T) {
|
||||
src := "--select * from dropall('resolvespec_login');\n" +
|
||||
"create or replace function resolvespec_login(\n" +
|
||||
"INOUT p_data jsonb, OUT p_success boolean, OUT p_error text)\n" +
|
||||
"language plpgsql volatile security definer\n" +
|
||||
"as $$\nbegin end;\n$$;\n"
|
||||
|
||||
want := "--select * from dropall('resolvespec_login');\n" +
|
||||
"CREATE OR REPLACE FUNCTION resolvespec_login(\n" +
|
||||
" INOUT p_data jsonb\n" +
|
||||
" ,OUT p_success boolean\n" +
|
||||
" ,OUT p_error text\n" +
|
||||
")\n" +
|
||||
"LANGUAGE plpgsql\n" +
|
||||
"VOLATILE\n" +
|
||||
"SECURITY DEFINER\n" +
|
||||
"AS\n" +
|
||||
"$$\nbegin end;\n$$;\n"
|
||||
|
||||
got := format(src)
|
||||
if got != want {
|
||||
t.Errorf("header format mismatch\n--- got ---\n%s\n--- want ---\n%s", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestIdempotentSmall(t *testing.T) {
|
||||
src := "create function f(a int,b text) returns void language sql as $$ select 1 $$;"
|
||||
once := format(src)
|
||||
twice := format(once)
|
||||
if once != twice {
|
||||
t.Errorf("not idempotent\n--- once ---\n%s\n--- twice ---\n%s", once, twice)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCorpusIdempotentAndSafe(t *testing.T) {
|
||||
dir := filepath.Join("..", "..", "testdata", "corpus")
|
||||
entries, err := os.ReadDir(dir)
|
||||
if err != nil {
|
||||
t.Skipf("no corpus: %v", err)
|
||||
}
|
||||
var seen int
|
||||
for _, e := range entries {
|
||||
if e.IsDir() || !strings.HasSuffix(e.Name(), ".pgsql") {
|
||||
continue
|
||||
}
|
||||
seen++
|
||||
data, err := os.ReadFile(filepath.Join(dir, e.Name()))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
src := string(data)
|
||||
once := format(src)
|
||||
twice := format(once)
|
||||
if once != twice {
|
||||
t.Errorf("%s: not idempotent", e.Name())
|
||||
}
|
||||
if !semanticallyEqual(src, once) {
|
||||
t.Errorf("%s: formatting changed semantics", e.Name())
|
||||
}
|
||||
}
|
||||
if seen == 0 {
|
||||
t.Skip("no corpus files")
|
||||
}
|
||||
t.Logf("formatted %d corpus files (idempotent + semantically equal)", seen)
|
||||
}
|
||||
|
||||
// semanticallyEqual compares the non-trivia token streams of two sources,
|
||||
// treating unquoted identifiers/keywords case-insensitively and everything
|
||||
// else (strings, numbers, dollar bodies, operators, punctuation) exactly. This
|
||||
// validates that formatting changed only layout/casing, never meaning.
|
||||
func semanticallyEqual(a, b string) bool {
|
||||
ta := significant(a)
|
||||
tb := significant(b)
|
||||
if len(ta) != len(tb) {
|
||||
return false
|
||||
}
|
||||
for i := range ta {
|
||||
if ta[i].Kind != tb[i].Kind {
|
||||
return false
|
||||
}
|
||||
if ta[i].Kind == lexer.Ident {
|
||||
if !strings.EqualFold(ta[i].Text, tb[i].Text) {
|
||||
return false
|
||||
}
|
||||
} else if ta[i].Text != tb[i].Text {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func significant(src string) []lexer.Token {
|
||||
var out []lexer.Token
|
||||
for _, t := range lexer.Lex(src) {
|
||||
if t.Kind == lexer.EOF || t.IsTrivia() {
|
||||
continue
|
||||
}
|
||||
out = append(out, t)
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,55 @@
|
||||
package format
|
||||
|
||||
// keywords are SQL / PL/pgSQL words the formatter may re-case via KeywordCase.
|
||||
// Type-name words are deliberately excluded (see typeNames) so the house style
|
||||
// can keep types lowercase while keywords are uppercased.
|
||||
var keywords = words(`
|
||||
add after all alter analyze and any array as asc atomic begin between by
|
||||
called cascade case cast check close coalesce collate column commit
|
||||
concurrently constraint create cross cube current_date current_time
|
||||
current_timestamp current_user cursor declare default deferrable definer delete desc
|
||||
distinct do drop each else elsif end except exception execute exists external
|
||||
fetch filter first for foreach foreign from full function get grant group
|
||||
grouping having if ilike immutable in index inner inout insert intersect into
|
||||
into invoker is join key language last leakproof left like limit localtime
|
||||
localtimestamp loop materialized natural new next no not nothing notify null
|
||||
nulls of off offset old on only open or order out outer over overriding
|
||||
parallel partition perform precision primary procedure raise
|
||||
references refresh rename replace reset restrict return returning returns
|
||||
revoke right rollback row rows safe schema secdef security select sequence set
|
||||
setof some stable stacked strict table temp temporary then to transaction
|
||||
trigger truncate union unique unsafe update using vacuum values variadic view
|
||||
volatile when where while window with within
|
||||
`)
|
||||
|
||||
// typeNames are built-in/common type words kept lowercase by the house style.
|
||||
var typeNames = words(`
|
||||
bigint bigserial bit bool boolean box bytea char character cidr circle citext
|
||||
date daterange decimal double float float4 float8 hstore inet int int2 int4
|
||||
int8 integer interval json jsonb line lseg macaddr macaddr8 money numeric oid
|
||||
path pg_lsn point polygon real serial serial2 serial4 serial8 smallint
|
||||
smallserial text time timestamp timestamptz timetz tsquery tsrange tstzrange
|
||||
tsvector uuid varbit varchar xml
|
||||
`)
|
||||
|
||||
func words(s string) map[string]bool {
|
||||
m := make(map[string]bool)
|
||||
w := ""
|
||||
for _, r := range s {
|
||||
if r == ' ' || r == '\n' || r == '\t' || r == '\r' {
|
||||
if w != "" {
|
||||
m[w] = true
|
||||
w = ""
|
||||
}
|
||||
continue
|
||||
}
|
||||
w += string(r)
|
||||
}
|
||||
if w != "" {
|
||||
m[w] = true
|
||||
}
|
||||
return m
|
||||
}
|
||||
|
||||
func isKeyword(lower string) bool { return keywords[lower] }
|
||||
func isTypeName(lower string) bool { return typeNames[lower] }
|
||||
@@ -0,0 +1,370 @@
|
||||
package lexer
|
||||
|
||||
import "strings"
|
||||
|
||||
// opChars are the characters PostgreSQL allows in operator names.
|
||||
const opChars = "+-*/<>=~!@#%^&|`?"
|
||||
|
||||
// opSpecial is the subset whose presence lets an operator end in + or -.
|
||||
const opSpecial = "~!@#%^&|`?"
|
||||
|
||||
// Lex tokenizes src into a lossless token stream. The concatenation of every
|
||||
// returned token's Text equals src. The final token is always EOF (empty Text).
|
||||
func Lex(src string) []Token {
|
||||
l := &lexer{src: src, line: 1, col: 1}
|
||||
return l.run()
|
||||
}
|
||||
|
||||
type lexer struct {
|
||||
src string
|
||||
pos int
|
||||
line int
|
||||
col int
|
||||
out []Token
|
||||
|
||||
// position of the token currently being scanned, captured each iteration
|
||||
tokOff, tokLine, tokCol int
|
||||
}
|
||||
|
||||
func (l *lexer) run() []Token {
|
||||
n := len(l.src)
|
||||
for l.pos < n {
|
||||
l.tokOff, l.tokLine, l.tokCol = l.pos, l.line, l.col
|
||||
start := l.pos
|
||||
c := l.src[l.pos]
|
||||
switch {
|
||||
case isSpace(c):
|
||||
l.scanWhile(isSpace)
|
||||
l.emit(Whitespace, start)
|
||||
case c == '-' && l.peek(1) == '-':
|
||||
l.scanLineComment()
|
||||
l.emit(LineComment, start)
|
||||
case c == '/' && l.peek(1) == '*':
|
||||
l.scanBlockComment()
|
||||
l.emit(BlockComment, start)
|
||||
case c == '\'':
|
||||
l.scanString('\'', false)
|
||||
l.emit(String, start)
|
||||
case c == '"':
|
||||
l.scanString('"', false)
|
||||
l.emit(QuotedIdent, start)
|
||||
case c == '$':
|
||||
l.scanDollar(start)
|
||||
case c == '(':
|
||||
l.advance(1)
|
||||
l.emit(LParen, start)
|
||||
case c == ')':
|
||||
l.advance(1)
|
||||
l.emit(RParen, start)
|
||||
case c == '[':
|
||||
l.advance(1)
|
||||
l.emit(LBracket, start)
|
||||
case c == ']':
|
||||
l.advance(1)
|
||||
l.emit(RBracket, start)
|
||||
case c == ',':
|
||||
l.advance(1)
|
||||
l.emit(Comma, start)
|
||||
case c == ';':
|
||||
l.advance(1)
|
||||
l.emit(Semicolon, start)
|
||||
case c == ':':
|
||||
// :: (cast), := (assign), or bare : (slice).
|
||||
if l.peek(1) == ':' || l.peek(1) == '=' {
|
||||
l.advance(2)
|
||||
} else {
|
||||
l.advance(1)
|
||||
}
|
||||
l.emit(Operator, start)
|
||||
case c == '.':
|
||||
if isDigit(l.peek(1)) {
|
||||
l.scanNumber()
|
||||
l.emit(Number, start)
|
||||
} else {
|
||||
l.advance(1)
|
||||
l.emit(Dot, start)
|
||||
}
|
||||
case isDigit(c):
|
||||
l.scanNumber()
|
||||
l.emit(Number, start)
|
||||
case isIdentStart(c):
|
||||
l.scanWord(start)
|
||||
case strings.IndexByte(opChars, c) >= 0:
|
||||
l.scanOperator()
|
||||
l.emit(Operator, start)
|
||||
default:
|
||||
l.advance(1)
|
||||
l.emit(Unknown, start)
|
||||
}
|
||||
}
|
||||
l.out = append(l.out, Token{Kind: EOF, Off: l.pos, Line: l.line, Col: l.col})
|
||||
return l.out
|
||||
}
|
||||
|
||||
// scanWord handles identifiers and the typed-string prefixes E' B' X' U&' / U&".
|
||||
func (l *lexer) scanWord(start int) {
|
||||
c := l.src[l.pos]
|
||||
switch c {
|
||||
case 'E', 'e':
|
||||
if l.peek(1) == '\'' {
|
||||
l.advance(1)
|
||||
l.scanString('\'', true)
|
||||
l.emit(EscapeString, start)
|
||||
return
|
||||
}
|
||||
case 'B', 'b':
|
||||
if l.peek(1) == '\'' {
|
||||
l.advance(1)
|
||||
l.scanString('\'', false)
|
||||
l.emit(BitString, start)
|
||||
return
|
||||
}
|
||||
case 'X', 'x':
|
||||
if l.peek(1) == '\'' {
|
||||
l.advance(1)
|
||||
l.scanString('\'', false)
|
||||
l.emit(HexString, start)
|
||||
return
|
||||
}
|
||||
case 'U', 'u':
|
||||
if l.peek(1) == '&' && (l.peek(2) == '\'' || l.peek(2) == '"') {
|
||||
q := l.peek(2)
|
||||
l.advance(2)
|
||||
l.scanString(q, false)
|
||||
l.emit(UnicodeString, start)
|
||||
return
|
||||
}
|
||||
}
|
||||
l.scanWhile(isIdentCont)
|
||||
l.emit(Ident, start)
|
||||
}
|
||||
|
||||
// scanDollar handles dollar-quoted strings ($tag$...$tag$), positional
|
||||
// parameters ($1), and a lone $.
|
||||
func (l *lexer) scanDollar(start int) {
|
||||
if tag, ok := l.dollarTag(); ok {
|
||||
// Opening delimiter is "$tag$"; find the matching close.
|
||||
open := "$" + tag + "$"
|
||||
l.advance(len(open))
|
||||
if idx := strings.Index(l.src[l.pos:], open); idx >= 0 {
|
||||
l.advance(idx + len(open))
|
||||
} else {
|
||||
l.advance(len(l.src) - l.pos) // unterminated: consume to EOF
|
||||
}
|
||||
l.emit(DollarString, start)
|
||||
return
|
||||
}
|
||||
if isDigit(l.peek(1)) {
|
||||
l.advance(1)
|
||||
l.scanWhile(isDigit)
|
||||
l.emit(Param, start)
|
||||
return
|
||||
}
|
||||
l.advance(1)
|
||||
l.emit(Unknown, start)
|
||||
}
|
||||
|
||||
// dollarTag checks whether the current $ begins a dollar-quote opening
|
||||
// delimiter and, if so, returns the (possibly empty) tag.
|
||||
func (l *lexer) dollarTag() (string, bool) {
|
||||
// src[pos] == '$'
|
||||
j := l.pos + 1
|
||||
n := len(l.src)
|
||||
if j < n && l.src[j] == '$' {
|
||||
return "", true // "$$"
|
||||
}
|
||||
k := j
|
||||
if k < n && isIdentStart(l.src[k]) {
|
||||
k++
|
||||
// Tag chars follow identifier rules but exclude '$' (which closes the tag).
|
||||
for k < n && isIdentCont(l.src[k]) && l.src[k] != '$' {
|
||||
k++
|
||||
}
|
||||
if k < n && l.src[k] == '$' {
|
||||
return l.src[j:k], true
|
||||
}
|
||||
}
|
||||
return "", false
|
||||
}
|
||||
|
||||
func (l *lexer) scanLineComment() {
|
||||
// Consume until newline (exclusive); the newline is whitespace.
|
||||
for l.pos < len(l.src) && l.src[l.pos] != '\n' {
|
||||
l.advance(1)
|
||||
}
|
||||
}
|
||||
|
||||
func (l *lexer) scanBlockComment() {
|
||||
l.advance(2) // /*
|
||||
depth := 1
|
||||
for l.pos < len(l.src) && depth > 0 {
|
||||
if l.src[l.pos] == '/' && l.peek(1) == '*' {
|
||||
l.advance(2)
|
||||
depth++
|
||||
} else if l.src[l.pos] == '*' && l.peek(1) == '/' {
|
||||
l.advance(2)
|
||||
depth--
|
||||
} else {
|
||||
l.advance(1)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// scanString consumes a quoted run beginning at the current quote character.
|
||||
// A doubled quote (”) is an embedded quote; with backslash=true a backslash
|
||||
// escapes the next byte (escape-string syntax).
|
||||
func (l *lexer) scanString(quote byte, backslash bool) {
|
||||
l.advance(1) // opening quote
|
||||
for l.pos < len(l.src) {
|
||||
c := l.src[l.pos]
|
||||
if backslash && c == '\\' && l.pos+1 < len(l.src) {
|
||||
l.advance(2)
|
||||
continue
|
||||
}
|
||||
if c == quote {
|
||||
if l.peek(1) == quote {
|
||||
l.advance(2) // doubled quote
|
||||
continue
|
||||
}
|
||||
l.advance(1) // closing quote
|
||||
return
|
||||
}
|
||||
l.advance(1)
|
||||
}
|
||||
}
|
||||
|
||||
func (l *lexer) scanNumber() {
|
||||
// Non-decimal integer literals: 0x.., 0o.., 0b..
|
||||
if l.src[l.pos] == '0' {
|
||||
switch l.peek(1) {
|
||||
case 'x', 'X', 'o', 'O', 'b', 'B':
|
||||
l.advance(2)
|
||||
l.scanWhile(isHexOrSep)
|
||||
return
|
||||
}
|
||||
}
|
||||
l.scanWhile(isDigitOrSep)
|
||||
if l.pos < len(l.src) && l.src[l.pos] == '.' {
|
||||
l.advance(1)
|
||||
l.scanWhile(isDigitOrSep)
|
||||
}
|
||||
if c := l.cur(); c == 'e' || c == 'E' {
|
||||
if p := l.peek(1); isDigit(p) || ((p == '+' || p == '-') && isDigit(l.peek(2))) {
|
||||
l.advance(1)
|
||||
if l.cur() == '+' || l.cur() == '-' {
|
||||
l.advance(1)
|
||||
}
|
||||
l.scanWhile(isDigit)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// scanOperator consumes a run of operator characters, applying PostgreSQL's
|
||||
// rule that a multi-character operator may not end in + or - unless it also
|
||||
// contains one of ~ ! @ # % ^ & | ` ?.
|
||||
func (l *lexer) scanOperator() {
|
||||
start := l.pos
|
||||
hasSpecial := false
|
||||
for l.pos < len(l.src) {
|
||||
c := l.src[l.pos]
|
||||
if strings.IndexByte(opChars, c) < 0 {
|
||||
break
|
||||
}
|
||||
if c == '-' && l.peek(1) == '-' {
|
||||
break // comment start
|
||||
}
|
||||
if c == '/' && l.peek(1) == '*' {
|
||||
break // comment start
|
||||
}
|
||||
if strings.IndexByte(opSpecial, c) >= 0 {
|
||||
hasSpecial = true
|
||||
}
|
||||
l.advance(1)
|
||||
}
|
||||
if !hasSpecial {
|
||||
for l.pos-1 > start {
|
||||
last := l.src[l.pos-1]
|
||||
if last != '+' && last != '-' {
|
||||
break
|
||||
}
|
||||
l.backup()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// --- low-level helpers ---
|
||||
|
||||
func (l *lexer) cur() byte {
|
||||
if l.pos < len(l.src) {
|
||||
return l.src[l.pos]
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
func (l *lexer) peek(k int) byte {
|
||||
if l.pos+k < len(l.src) {
|
||||
return l.src[l.pos+k]
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
func (l *lexer) scanWhile(pred func(byte) bool) {
|
||||
for l.pos < len(l.src) && pred(l.src[l.pos]) {
|
||||
l.advance(1)
|
||||
}
|
||||
}
|
||||
|
||||
// advance moves forward k bytes, maintaining line/col.
|
||||
func (l *lexer) advance(k int) {
|
||||
for i := 0; i < k && l.pos < len(l.src); i++ {
|
||||
if l.src[l.pos] == '\n' {
|
||||
l.line++
|
||||
l.col = 1
|
||||
} else {
|
||||
l.col++
|
||||
}
|
||||
l.pos++
|
||||
}
|
||||
}
|
||||
|
||||
// backup moves back one byte. Only used within scanOperator, which never
|
||||
// spans a newline, so column bookkeeping is safe.
|
||||
func (l *lexer) backup() {
|
||||
l.pos--
|
||||
l.col--
|
||||
}
|
||||
|
||||
// emit appends a token covering src[start:pos], using the start position
|
||||
// captured at the top of the scan loop.
|
||||
func (l *lexer) emit(kind Kind, start int) {
|
||||
l.out = append(l.out, Token{
|
||||
Kind: kind,
|
||||
Text: l.src[start:l.pos],
|
||||
Off: l.tokOff,
|
||||
Line: l.tokLine,
|
||||
Col: l.tokCol,
|
||||
})
|
||||
}
|
||||
|
||||
func isSpace(c byte) bool {
|
||||
switch c {
|
||||
case ' ', '\t', '\n', '\r', '\v', '\f':
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func isDigit(c byte) bool { return c >= '0' && c <= '9' }
|
||||
func isDigitOrSep(c byte) bool { return isDigit(c) || c == '_' }
|
||||
|
||||
func isHexOrSep(c byte) bool {
|
||||
return isDigit(c) || (c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F') || c == '_'
|
||||
}
|
||||
|
||||
func isIdentStart(c byte) bool {
|
||||
return c == '_' || (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c >= 0x80
|
||||
}
|
||||
|
||||
func isIdentCont(c byte) bool {
|
||||
return isIdentStart(c) || isDigit(c) || c == '$'
|
||||
}
|
||||
@@ -0,0 +1,146 @@
|
||||
package lexer
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// emit reconstructs the source from a token stream.
|
||||
func emit(toks []Token) string {
|
||||
var b strings.Builder
|
||||
for _, t := range toks {
|
||||
b.WriteString(t.Text)
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
|
||||
func TestRoundTripSmall(t *testing.T) {
|
||||
cases := []string{
|
||||
"",
|
||||
"SELECT 1;",
|
||||
"select * from t where a = b;",
|
||||
"-- a comment\nSELECT 1",
|
||||
"/* block /* nested */ still */ SELECT 1",
|
||||
"SELECT 'it''s', E'a\\nb', $$dollar$$, $tag$x$tag$, $1;",
|
||||
"a->>'b'::text",
|
||||
"x := y + 1;",
|
||||
"a=-b",
|
||||
"SELECT 1.5, .5, 1e10, 0xFF, 1_000;",
|
||||
"arr[1:2]",
|
||||
"\"Quoted Ident\".col",
|
||||
"U&'d\\0061t'",
|
||||
}
|
||||
for _, src := range cases {
|
||||
got := emit(Lex(src))
|
||||
if got != src {
|
||||
t.Errorf("round-trip mismatch\n in: %q\nout: %q", src, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestKinds(t *testing.T) {
|
||||
toks := nonEOF(Lex("a->>'b'::text"))
|
||||
want := []Kind{Ident, Operator, String, Operator, Ident}
|
||||
if len(toks) != len(want) {
|
||||
t.Fatalf("got %d tokens, want %d: %v", len(toks), len(want), toks)
|
||||
}
|
||||
for i, k := range want {
|
||||
if toks[i].Kind != k {
|
||||
t.Errorf("token %d: got %s, want %s (text %q)", i, toks[i].Kind, k, toks[i].Text)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestOperatorTrailingRule(t *testing.T) {
|
||||
// "=-" must split into "=" and "-" (no special char, can't end in -).
|
||||
toks := nonEOF(Lex("a=-b"))
|
||||
if len(toks) != 4 || toks[1].Text != "=" || toks[2].Text != "-" {
|
||||
t.Fatalf("a=-b mis-lexed: %v", toks)
|
||||
}
|
||||
// "@-" keeps trailing - because @ is special.
|
||||
toks = nonEOF(Lex("a@-b"))
|
||||
if toks[1].Text != "@-" {
|
||||
t.Fatalf("@- should stay one operator, got %q", toks[1].Text)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDollarQuote(t *testing.T) {
|
||||
toks := nonEOF(Lex("$func$ body $$ inner $func$"))
|
||||
if len(toks) != 1 || toks[0].Kind != DollarString {
|
||||
t.Fatalf("dollar quote not single token: %v", toks)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLineColumns(t *testing.T) {
|
||||
toks := nonEOF(Lex("ab\n cd"))
|
||||
// ab(1,1) ws cd(2,3)
|
||||
if toks[0].Line != 1 || toks[0].Col != 1 {
|
||||
t.Errorf("ab at %d:%d, want 1:1", toks[0].Line, toks[0].Col)
|
||||
}
|
||||
cd := toks[len(toks)-1]
|
||||
if cd.Text != "cd" || cd.Line != 2 || cd.Col != 3 {
|
||||
t.Errorf("cd at %d:%d, want 2:3", cd.Line, cd.Col)
|
||||
}
|
||||
}
|
||||
|
||||
// TestCorpusRoundTrip is the core lossless invariant: lexing then re-emitting
|
||||
// every real-world .pgsql fixture must reproduce it byte-for-byte.
|
||||
func TestCorpusRoundTrip(t *testing.T) {
|
||||
dir := filepath.Join("..", "..", "testdata", "corpus")
|
||||
entries, err := os.ReadDir(dir)
|
||||
if err != nil {
|
||||
t.Skipf("no corpus dir: %v", err)
|
||||
}
|
||||
var seen int
|
||||
for _, e := range entries {
|
||||
if e.IsDir() || !strings.HasSuffix(e.Name(), ".pgsql") {
|
||||
continue
|
||||
}
|
||||
seen++
|
||||
path := filepath.Join(dir, e.Name())
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
t.Fatalf("read %s: %v", path, err)
|
||||
}
|
||||
src := string(data)
|
||||
if got := emit(Lex(src)); got != src {
|
||||
t.Errorf("%s: round-trip mismatch (len in=%d out=%d)", e.Name(), len(src), len(got))
|
||||
reportFirstDiff(t, e.Name(), src, got)
|
||||
}
|
||||
}
|
||||
if seen == 0 {
|
||||
t.Skip("corpus dir has no .pgsql files")
|
||||
}
|
||||
t.Logf("round-tripped %d corpus files", seen)
|
||||
}
|
||||
|
||||
func reportFirstDiff(t *testing.T, name, a, b string) {
|
||||
t.Helper()
|
||||
n := len(a)
|
||||
if len(b) < n {
|
||||
n = len(b)
|
||||
}
|
||||
for i := 0; i < n; i++ {
|
||||
if a[i] != b[i] {
|
||||
lo := i - 20
|
||||
if lo < 0 {
|
||||
lo = 0
|
||||
}
|
||||
t.Logf("%s: first diff at byte %d\n in: %q\n out: %q", name, i, a[lo:min(i+20, len(a))], b[lo:min(i+20, len(b))])
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func nonEOF(toks []Token) []Token {
|
||||
var out []Token
|
||||
for _, t := range toks {
|
||||
if t.Kind == EOF {
|
||||
continue
|
||||
}
|
||||
out = append(out, t)
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,102 @@
|
||||
// Package lexer implements a lossless lexer for PostgreSQL SQL and PL/pgSQL.
|
||||
//
|
||||
// "Lossless" means every byte of the input is represented by exactly one token,
|
||||
// including whitespace and comments. Concatenating the Text of all tokens in
|
||||
// order reproduces the original source byte-for-byte:
|
||||
//
|
||||
// emit(Lex(src)) == src
|
||||
//
|
||||
// This property is the foundation of the formatter: comments and whitespace are
|
||||
// first-class tokens (trivia) so the parser/printer can preserve them.
|
||||
package lexer
|
||||
|
||||
// Kind classifies a token.
|
||||
type Kind int
|
||||
|
||||
const (
|
||||
EOF Kind = iota
|
||||
|
||||
// Trivia — insignificant to the grammar but preserved losslessly.
|
||||
Whitespace
|
||||
LineComment // -- ... (up to, not including, the newline)
|
||||
BlockComment // /* ... */ (nestable)
|
||||
|
||||
// Words.
|
||||
Ident // unquoted identifier or keyword (keyword-ness resolved later)
|
||||
QuotedIdent // "..."
|
||||
|
||||
// Literals.
|
||||
String // '...'
|
||||
EscapeString // E'...'
|
||||
BitString // B'...'
|
||||
HexString // X'...'
|
||||
UnicodeString // U&'...' or U&"..."
|
||||
DollarString // $tag$...$tag$
|
||||
Number // 123, 1.5, .5, 1e10, 0xff, 1_000
|
||||
Param // $1, $2
|
||||
|
||||
// Operators (incl. ::, :=, :, ->, ->>, and op-char runs).
|
||||
Operator
|
||||
|
||||
// Structural punctuation.
|
||||
LParen
|
||||
RParen
|
||||
LBracket
|
||||
RBracket
|
||||
Comma
|
||||
Semicolon
|
||||
Dot
|
||||
|
||||
Unknown // a byte that fits no other category
|
||||
)
|
||||
|
||||
var kindNames = map[Kind]string{
|
||||
EOF: "EOF",
|
||||
Whitespace: "Whitespace",
|
||||
LineComment: "LineComment",
|
||||
BlockComment: "BlockComment",
|
||||
Ident: "Ident",
|
||||
QuotedIdent: "QuotedIdent",
|
||||
String: "String",
|
||||
EscapeString: "EscapeString",
|
||||
BitString: "BitString",
|
||||
HexString: "HexString",
|
||||
UnicodeString: "UnicodeString",
|
||||
DollarString: "DollarString",
|
||||
Number: "Number",
|
||||
Param: "Param",
|
||||
Operator: "Operator",
|
||||
LParen: "LParen",
|
||||
RParen: "RParen",
|
||||
LBracket: "LBracket",
|
||||
RBracket: "RBracket",
|
||||
Comma: "Comma",
|
||||
Semicolon: "Semicolon",
|
||||
Dot: "Dot",
|
||||
Unknown: "Unknown",
|
||||
}
|
||||
|
||||
func (k Kind) String() string {
|
||||
if s, ok := kindNames[k]; ok {
|
||||
return s
|
||||
}
|
||||
return "Kind(?)"
|
||||
}
|
||||
|
||||
// Token is a single lexical unit covering Text == src[Off:Off+len(Text)].
|
||||
type Token struct {
|
||||
Kind Kind
|
||||
Text string
|
||||
Off int // byte offset of the first byte
|
||||
Line int // 1-based line of the first byte
|
||||
Col int // 1-based byte column of the first byte
|
||||
}
|
||||
|
||||
// IsTrivia reports whether the token is whitespace or a comment.
|
||||
func (t Token) IsTrivia() bool {
|
||||
switch t.Kind {
|
||||
case Whitespace, LineComment, BlockComment:
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
@@ -0,0 +1,263 @@
|
||||
// Package parser builds a cst.File from PostgreSQL source.
|
||||
//
|
||||
// The parser is intentionally partial: it splits input into statements and
|
||||
// structures the constructs the formatter currently understands (today:
|
||||
// CREATE FUNCTION/PROCEDURE). Anything else is preserved verbatim as a cst.Raw
|
||||
// node. This guarantees the parser never loses or corrupts input — see the
|
||||
// round-trip test.
|
||||
package parser
|
||||
|
||||
import (
|
||||
"github.com/hein/pgtidy/pkg/cst"
|
||||
"github.com/hein/pgtidy/pkg/lexer"
|
||||
)
|
||||
|
||||
// Parse lexes and parses src into a lossless cst.File.
|
||||
func Parse(src string) *cst.File {
|
||||
sig, trailing := cst.Attach(lexer.Lex(src))
|
||||
f := &cst.File{Trailing: trailing}
|
||||
for _, stmt := range splitStatements(sig) {
|
||||
f.Items = append(f.Items, parseStatement(stmt))
|
||||
}
|
||||
return f
|
||||
}
|
||||
|
||||
// splitStatements breaks the significant-token stream into statements at
|
||||
// top-level (paren-depth 0) semicolons. The terminating semicolon is included
|
||||
// in the statement it ends.
|
||||
func splitStatements(sig []cst.Tok) [][]cst.Tok {
|
||||
var stmts [][]cst.Tok
|
||||
var cur []cst.Tok
|
||||
depth := 0
|
||||
for _, t := range sig {
|
||||
cur = append(cur, t)
|
||||
switch t.Tok.Kind {
|
||||
case lexer.LParen:
|
||||
depth++
|
||||
case lexer.RParen:
|
||||
if depth > 0 {
|
||||
depth--
|
||||
}
|
||||
case lexer.Semicolon:
|
||||
if depth == 0 {
|
||||
stmts = append(stmts, cur)
|
||||
cur = nil
|
||||
}
|
||||
}
|
||||
}
|
||||
if len(cur) > 0 {
|
||||
stmts = append(stmts, cur)
|
||||
}
|
||||
return stmts
|
||||
}
|
||||
|
||||
func parseStatement(stmt []cst.Tok) cst.Node {
|
||||
if isCreateFunction(stmt) {
|
||||
if cf, ok := parseCreateFunction(stmt); ok {
|
||||
return cf
|
||||
}
|
||||
}
|
||||
return &cst.Raw{Toks: stmt}
|
||||
}
|
||||
|
||||
// isCreateFunction reports whether stmt begins with CREATE ... FUNCTION|PROCEDURE.
|
||||
func isCreateFunction(stmt []cst.Tok) bool {
|
||||
if len(stmt) == 0 || !stmt[0].Is("create") {
|
||||
return false
|
||||
}
|
||||
for i := 1; i < len(stmt) && i < 4; i++ {
|
||||
if stmt[i].Is("function") || stmt[i].Is("procedure") {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// parseCreateFunction structures a CREATE FUNCTION/PROCEDURE statement. It
|
||||
// returns ok=false (so the caller falls back to Raw) if the shape is not what
|
||||
// it expects.
|
||||
func parseCreateFunction(stmt []cst.Tok) (*cst.CreateFunction, bool) {
|
||||
cf := &cst.CreateFunction{}
|
||||
i := 0
|
||||
|
||||
// Head: CREATE [OR REPLACE] (FUNCTION|PROCEDURE)
|
||||
if i >= len(stmt) || !stmt[i].Is("create") {
|
||||
return nil, false
|
||||
}
|
||||
i++
|
||||
if i+1 < len(stmt) && stmt[i].Is("or") && stmt[i+1].Is("replace") {
|
||||
i += 2
|
||||
}
|
||||
if i >= len(stmt) || !(stmt[i].Is("function") || stmt[i].Is("procedure")) {
|
||||
return nil, false
|
||||
}
|
||||
i++
|
||||
cf.Head = stmt[:i]
|
||||
|
||||
// Name: everything up to the opening '(' of the parameter list.
|
||||
nameStart := i
|
||||
for i < len(stmt) && stmt[i].Tok.Kind != lexer.LParen {
|
||||
i++
|
||||
}
|
||||
if i >= len(stmt) {
|
||||
return nil, false // no parameter list
|
||||
}
|
||||
cf.Name = stmt[nameStart:i]
|
||||
cf.LParen = stmt[i]
|
||||
i++
|
||||
|
||||
// Parameters: up to the matching ')'.
|
||||
params, rparen, ok := parseParamList(stmt, i)
|
||||
if !ok {
|
||||
return nil, false
|
||||
}
|
||||
cf.Params = params
|
||||
cf.RParen = stmt[rparen]
|
||||
i = rparen + 1
|
||||
|
||||
// Trailing semicolon (handle now so it isn't swept into options/tail).
|
||||
end := len(stmt)
|
||||
if end > 0 && stmt[end-1].Tok.Kind == lexer.Semicolon {
|
||||
cf.Semi = &stmt[end-1]
|
||||
end--
|
||||
}
|
||||
|
||||
// Options up to AS; then the body; then any tail options.
|
||||
asIdx := indexOfKeyword(stmt, i, end, "as")
|
||||
if asIdx < 0 {
|
||||
// No AS clause (e.g. RETURN / BEGIN ATOMIC bodies). Keep options
|
||||
// structured but leave the body unstructured.
|
||||
cf.Options = splitClauses(stmt[i:end])
|
||||
return cf, true
|
||||
}
|
||||
cf.Options = splitClauses(stmt[i:asIdx])
|
||||
cf.As = &stmt[asIdx]
|
||||
bi := asIdx + 1
|
||||
if bi < end && isBodyToken(stmt[bi]) {
|
||||
cf.Body = &stmt[bi]
|
||||
bi++
|
||||
}
|
||||
if bi < end {
|
||||
cf.Tail = splitClauses(stmt[bi:end])
|
||||
}
|
||||
return cf, true
|
||||
}
|
||||
|
||||
// parseParamList parses comma-separated parameters starting at index `start`
|
||||
// (the token after '('). It returns the parameters, the index of the matching
|
||||
// ')', and ok.
|
||||
func parseParamList(stmt []cst.Tok, start int) ([]cst.Param, int, bool) {
|
||||
var params []cst.Param
|
||||
depth := 0
|
||||
itemStart := start
|
||||
i := start
|
||||
flush := func(endExclusive int, sep *cst.Tok) {
|
||||
toks := stmt[itemStart:endExclusive]
|
||||
if len(toks) == 0 && sep == nil {
|
||||
return // empty parameter list ()
|
||||
}
|
||||
params = append(params, cst.Param{Toks: toks, Sep: sep})
|
||||
}
|
||||
for ; i < len(stmt); i++ {
|
||||
switch stmt[i].Tok.Kind {
|
||||
case lexer.LParen, lexer.LBracket:
|
||||
depth++
|
||||
case lexer.RParen:
|
||||
if depth == 0 {
|
||||
// Closing the parameter list.
|
||||
flush(i, nil)
|
||||
return params, i, true
|
||||
}
|
||||
depth--
|
||||
case lexer.RBracket:
|
||||
if depth > 0 {
|
||||
depth--
|
||||
}
|
||||
case lexer.Comma:
|
||||
if depth == 0 {
|
||||
sep := &stmt[i]
|
||||
flush(i, sep)
|
||||
itemStart = i + 1
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil, 0, false // unbalanced
|
||||
}
|
||||
|
||||
// splitClauses groups a run of option tokens into clauses, each beginning at a
|
||||
// recognized clause keyword (at paren depth 0). Tokens before the first
|
||||
// recognized keyword, if any, are attached to a leading clause so nothing is
|
||||
// dropped.
|
||||
func splitClauses(toks []cst.Tok) [][]cst.Tok {
|
||||
if len(toks) == 0 {
|
||||
return nil
|
||||
}
|
||||
var clauses [][]cst.Tok
|
||||
depth := 0
|
||||
start := 0
|
||||
for i, t := range toks {
|
||||
switch t.Tok.Kind {
|
||||
case lexer.LParen, lexer.LBracket:
|
||||
depth++
|
||||
case lexer.RParen, lexer.RBracket:
|
||||
if depth > 0 {
|
||||
depth--
|
||||
}
|
||||
}
|
||||
if i > start && depth == 0 && isClauseStarter(t) {
|
||||
clauses = append(clauses, toks[start:i])
|
||||
start = i
|
||||
}
|
||||
}
|
||||
clauses = append(clauses, toks[start:])
|
||||
return clauses
|
||||
}
|
||||
|
||||
var clauseStarters = map[string]bool{
|
||||
"returns": true, "language": true, "transform": true, "window": true,
|
||||
"immutable": true, "stable": true, "volatile": true, "leakproof": true,
|
||||
"not": true, "called": true, "strict": true, "external": true,
|
||||
"security": true, "parallel": true, "cost": true, "rows": true,
|
||||
"support": true, "set": true, "as": true,
|
||||
}
|
||||
|
||||
func isClauseStarter(t cst.Tok) bool {
|
||||
if t.Tok.Kind != lexer.Ident {
|
||||
return false
|
||||
}
|
||||
return clauseStarters[lowerASCII(t.Tok.Text)]
|
||||
}
|
||||
|
||||
func isBodyToken(t cst.Tok) bool {
|
||||
return t.Tok.Kind == lexer.DollarString || t.Tok.Kind == lexer.String
|
||||
}
|
||||
|
||||
// indexOfKeyword returns the index in [from,to) of the first top-level (paren
|
||||
// depth 0) token equal to kw, or -1.
|
||||
func indexOfKeyword(stmt []cst.Tok, from, to int, kw string) int {
|
||||
depth := 0
|
||||
for i := from; i < to; i++ {
|
||||
switch stmt[i].Tok.Kind {
|
||||
case lexer.LParen, lexer.LBracket:
|
||||
depth++
|
||||
case lexer.RParen, lexer.RBracket:
|
||||
if depth > 0 {
|
||||
depth--
|
||||
}
|
||||
}
|
||||
if depth == 0 && stmt[i].Is(kw) {
|
||||
return i
|
||||
}
|
||||
}
|
||||
return -1
|
||||
}
|
||||
|
||||
func lowerASCII(s string) string {
|
||||
b := []byte(s)
|
||||
for i, c := range b {
|
||||
if c >= 'A' && c <= 'Z' {
|
||||
b[i] = c + 32
|
||||
}
|
||||
}
|
||||
return string(b)
|
||||
}
|
||||
@@ -0,0 +1,124 @@
|
||||
package parser
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/hein/pgtidy/pkg/cst"
|
||||
)
|
||||
|
||||
func TestParseRoundTripSmall(t *testing.T) {
|
||||
cases := []string{
|
||||
"",
|
||||
"SELECT 1;",
|
||||
"-- lead\nSELECT 1; SELECT 2;",
|
||||
"CREATE FUNCTION f() RETURNS int LANGUAGE sql AS $$ SELECT 1 $$;",
|
||||
"create or replace function s.g(a int, b text default 'x') returns void language plpgsql as $$ begin end; $$;\n",
|
||||
"INSERT INTO t (a,b) VALUES (1,2); -- trailing\n",
|
||||
}
|
||||
for _, src := range cases {
|
||||
if got := Parse(src).Source(); got != src {
|
||||
t.Errorf("round-trip mismatch\n in: %q\nout: %q", src, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseCreateFunctionShape(t *testing.T) {
|
||||
src := "CREATE OR REPLACE FUNCTION resolvespec_login(\n" +
|
||||
" INOUT p_data jsonb\n" +
|
||||
" ,OUT p_success boolean\n" +
|
||||
" ,OUT p_error text\n" +
|
||||
")\nLANGUAGE plpgsql VOLATILE\nSECURITY DEFINER\nAS\n$$\nbegin end;\n$$;\n"
|
||||
|
||||
f := Parse(src)
|
||||
if len(f.Items) != 1 {
|
||||
t.Fatalf("want 1 item, got %d", len(f.Items))
|
||||
}
|
||||
cf, ok := f.Items[0].(*cst.CreateFunction)
|
||||
if !ok {
|
||||
t.Fatalf("want *CreateFunction, got %T", f.Items[0])
|
||||
}
|
||||
if cf.IsProcedure() {
|
||||
t.Error("should be FUNCTION not PROCEDURE")
|
||||
}
|
||||
if len(cf.Params) != 3 {
|
||||
t.Fatalf("want 3 params, got %d", len(cf.Params))
|
||||
}
|
||||
// Leading-comma style: in source the comma precedes the next param, but the
|
||||
// parser associates each comma as the separator after the preceding param.
|
||||
if cf.Params[0].Sep == nil || cf.Params[1].Sep == nil || cf.Params[2].Sep != nil {
|
||||
t.Errorf("param separators wrong: %v %v %v",
|
||||
cf.Params[0].Sep != nil, cf.Params[1].Sep != nil, cf.Params[2].Sep != nil)
|
||||
}
|
||||
if cf.Body == nil || !strings.Contains(cf.Body.Text(), "begin") {
|
||||
t.Errorf("body not captured: %+v", cf.Body)
|
||||
}
|
||||
if cf.As == nil {
|
||||
t.Error("AS not captured")
|
||||
}
|
||||
// Options should include a LANGUAGE clause, a VOLATILE clause and a SECURITY clause.
|
||||
var langs, vols, secs int
|
||||
for _, cl := range cf.Options {
|
||||
if len(cl) == 0 {
|
||||
continue
|
||||
}
|
||||
switch {
|
||||
case cl[0].Is("language"):
|
||||
langs++
|
||||
case cl[0].Is("volatile"):
|
||||
vols++
|
||||
case cl[0].Is("security"):
|
||||
secs++
|
||||
}
|
||||
}
|
||||
if langs != 1 || vols != 1 || secs != 1 {
|
||||
t.Errorf("clause split wrong: language=%d volatile=%d security=%d", langs, vols, secs)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseFallbackRaw(t *testing.T) {
|
||||
f := Parse("SELECT a, b FROM t WHERE c = 1;")
|
||||
if len(f.Items) != 1 {
|
||||
t.Fatalf("want 1 item, got %d", len(f.Items))
|
||||
}
|
||||
if _, ok := f.Items[0].(*cst.Raw); !ok {
|
||||
t.Fatalf("want *Raw, got %T", f.Items[0])
|
||||
}
|
||||
}
|
||||
|
||||
// TestCorpusRoundTrip proves the parser is lossless on real-world input: the
|
||||
// reconstructed source must equal the original byte-for-byte.
|
||||
func TestCorpusRoundTrip(t *testing.T) {
|
||||
dir := filepath.Join("..", "..", "testdata", "corpus")
|
||||
entries, err := os.ReadDir(dir)
|
||||
if err != nil {
|
||||
t.Skipf("no corpus dir: %v", err)
|
||||
}
|
||||
var seen, fns int
|
||||
for _, e := range entries {
|
||||
if e.IsDir() || !strings.HasSuffix(e.Name(), ".pgsql") {
|
||||
continue
|
||||
}
|
||||
seen++
|
||||
data, err := os.ReadFile(filepath.Join(dir, e.Name()))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
src := string(data)
|
||||
f := Parse(src)
|
||||
if got := f.Source(); got != src {
|
||||
t.Errorf("%s: round-trip mismatch (len in=%d out=%d)", e.Name(), len(src), len(got))
|
||||
}
|
||||
for _, it := range f.Items {
|
||||
if _, ok := it.(*cst.CreateFunction); ok {
|
||||
fns++
|
||||
}
|
||||
}
|
||||
}
|
||||
if seen == 0 {
|
||||
t.Skip("no corpus files")
|
||||
}
|
||||
t.Logf("round-tripped %d corpus files; structured %d CREATE FUNCTION/PROCEDURE", seen, fns)
|
||||
}
|
||||
Reference in New Issue
Block a user