103 lines
2.5 KiB
Go
103 lines
2.5 KiB
Go
// Package lexer implements a lossless lexer for PostgreSQL SQL and PL/pgSQL.
|
|
//
|
|
// "Lossless" means every byte of the input is represented by exactly one token,
|
|
// including whitespace and comments. Concatenating the Text of all tokens in
|
|
// order reproduces the original source byte-for-byte:
|
|
//
|
|
// emit(Lex(src)) == src
|
|
//
|
|
// This property is the foundation of the formatter: comments and whitespace are
|
|
// first-class tokens (trivia) so the parser/printer can preserve them.
|
|
package lexer
|
|
|
|
// Kind classifies a token.
|
|
type Kind int
|
|
|
|
const (
|
|
EOF Kind = iota
|
|
|
|
// Trivia — insignificant to the grammar but preserved losslessly.
|
|
Whitespace
|
|
LineComment // -- ... (up to, not including, the newline)
|
|
BlockComment // /* ... */ (nestable)
|
|
|
|
// Words.
|
|
Ident // unquoted identifier or keyword (keyword-ness resolved later)
|
|
QuotedIdent // "..."
|
|
|
|
// Literals.
|
|
String // '...'
|
|
EscapeString // E'...'
|
|
BitString // B'...'
|
|
HexString // X'...'
|
|
UnicodeString // U&'...' or U&"..."
|
|
DollarString // $tag$...$tag$
|
|
Number // 123, 1.5, .5, 1e10, 0xff, 1_000
|
|
Param // $1, $2
|
|
|
|
// Operators (incl. ::, :=, :, ->, ->>, and op-char runs).
|
|
Operator
|
|
|
|
// Structural punctuation.
|
|
LParen
|
|
RParen
|
|
LBracket
|
|
RBracket
|
|
Comma
|
|
Semicolon
|
|
Dot
|
|
|
|
Unknown // a byte that fits no other category
|
|
)
|
|
|
|
var kindNames = map[Kind]string{
|
|
EOF: "EOF",
|
|
Whitespace: "Whitespace",
|
|
LineComment: "LineComment",
|
|
BlockComment: "BlockComment",
|
|
Ident: "Ident",
|
|
QuotedIdent: "QuotedIdent",
|
|
String: "String",
|
|
EscapeString: "EscapeString",
|
|
BitString: "BitString",
|
|
HexString: "HexString",
|
|
UnicodeString: "UnicodeString",
|
|
DollarString: "DollarString",
|
|
Number: "Number",
|
|
Param: "Param",
|
|
Operator: "Operator",
|
|
LParen: "LParen",
|
|
RParen: "RParen",
|
|
LBracket: "LBracket",
|
|
RBracket: "RBracket",
|
|
Comma: "Comma",
|
|
Semicolon: "Semicolon",
|
|
Dot: "Dot",
|
|
Unknown: "Unknown",
|
|
}
|
|
|
|
func (k Kind) String() string {
|
|
if s, ok := kindNames[k]; ok {
|
|
return s
|
|
}
|
|
return "Kind(?)"
|
|
}
|
|
|
|
// Token is a single lexical unit covering Text == src[Off:Off+len(Text)].
|
|
type Token struct {
|
|
Kind Kind
|
|
Text string
|
|
Off int // byte offset of the first byte
|
|
Line int // 1-based line of the first byte
|
|
Col int // 1-based byte column of the first byte
|
|
}
|
|
|
|
// IsTrivia reports whether the token is whitespace or a comment.
|
|
func (t Token) IsTrivia() bool {
|
|
switch t.Kind {
|
|
case Whitespace, LineComment, BlockComment:
|
|
return true
|
|
}
|
|
return false
|
|
}
|