// Package lexer implements a lossless lexer for PostgreSQL SQL and PL/pgSQL. // // "Lossless" means every byte of the input is represented by exactly one token, // including whitespace and comments. Concatenating the Text of all tokens in // order reproduces the original source byte-for-byte: // // emit(Lex(src)) == src // // This property is the foundation of the formatter: comments and whitespace are // first-class tokens (trivia) so the parser/printer can preserve them. package lexer // Kind classifies a token. type Kind int const ( EOF Kind = iota // Trivia — insignificant to the grammar but preserved losslessly. Whitespace LineComment // -- ... (up to, not including, the newline) BlockComment // /* ... */ (nestable) // Words. Ident // unquoted identifier or keyword (keyword-ness resolved later) QuotedIdent // "..." // Literals. String // '...' EscapeString // E'...' BitString // B'...' HexString // X'...' UnicodeString // U&'...' or U&"..." DollarString // $tag$...$tag$ Number // 123, 1.5, .5, 1e10, 0xff, 1_000 Param // $1, $2 // Operators (incl. ::, :=, :, ->, ->>, and op-char runs). Operator // Structural punctuation. LParen RParen LBracket RBracket Comma Semicolon Dot Unknown // a byte that fits no other category ) var kindNames = map[Kind]string{ EOF: "EOF", Whitespace: "Whitespace", LineComment: "LineComment", BlockComment: "BlockComment", Ident: "Ident", QuotedIdent: "QuotedIdent", String: "String", EscapeString: "EscapeString", BitString: "BitString", HexString: "HexString", UnicodeString: "UnicodeString", DollarString: "DollarString", Number: "Number", Param: "Param", Operator: "Operator", LParen: "LParen", RParen: "RParen", LBracket: "LBracket", RBracket: "RBracket", Comma: "Comma", Semicolon: "Semicolon", Dot: "Dot", Unknown: "Unknown", } func (k Kind) String() string { if s, ok := kindNames[k]; ok { return s } return "Kind(?)" } // Token is a single lexical unit covering Text == src[Off:Off+len(Text)]. type Token struct { Kind Kind Text string Off int // byte offset of the first byte Line int // 1-based line of the first byte Col int // 1-based byte column of the first byte } // IsTrivia reports whether the token is whitespace or a comment. func (t Token) IsTrivia() bool { switch t.Kind { case Whitespace, LineComment, BlockComment: return true } return false }