feat: initial plan

pkg/lexer/token.go

@@ -0,0 +1,102 @@
+// Package lexer implements a lossless lexer for PostgreSQL SQL and PL/pgSQL.
+//
+// "Lossless" means every byte of the input is represented by exactly one token,
+// including whitespace and comments. Concatenating the Text of all tokens in
+// order reproduces the original source byte-for-byte:
+//
+//	emit(Lex(src)) == src
+//
+// This property is the foundation of the formatter: comments and whitespace are
+// first-class tokens (trivia) so the parser/printer can preserve them.
+package lexer
+
+// Kind classifies a token.
+type Kind int
+
+const (
+	EOF Kind = iota
+
+	// Trivia — insignificant to the grammar but preserved losslessly.
+	Whitespace
+	LineComment  // -- ... (up to, not including, the newline)
+	BlockComment // /* ... */ (nestable)
+
+	// Words.
+	Ident       // unquoted identifier or keyword (keyword-ness resolved later)
+	QuotedIdent // "..."
+
+	// Literals.
+	String        // '...'
+	EscapeString  // E'...'
+	BitString     // B'...'
+	HexString     // X'...'
+	UnicodeString // U&'...' or U&"..."
+	DollarString  // $tag$...$tag$
+	Number        // 123, 1.5, .5, 1e10, 0xff, 1_000
+	Param         // $1, $2
+
+	// Operators (incl. ::, :=, :, ->, ->>, and op-char runs).
+	Operator
+
+	// Structural punctuation.
+	LParen
+	RParen
+	LBracket
+	RBracket
+	Comma
+	Semicolon
+	Dot
+
+	Unknown // a byte that fits no other category
+)
+
+var kindNames = map[Kind]string{
+	EOF:           "EOF",
+	Whitespace:    "Whitespace",
+	LineComment:   "LineComment",
+	BlockComment:  "BlockComment",
+	Ident:         "Ident",
+	QuotedIdent:   "QuotedIdent",
+	String:        "String",
+	EscapeString:  "EscapeString",
+	BitString:     "BitString",
+	HexString:     "HexString",
+	UnicodeString: "UnicodeString",
+	DollarString:  "DollarString",
+	Number:        "Number",
+	Param:         "Param",
+	Operator:      "Operator",
+	LParen:        "LParen",
+	RParen:        "RParen",
+	LBracket:      "LBracket",
+	RBracket:      "RBracket",
+	Comma:         "Comma",
+	Semicolon:     "Semicolon",
+	Dot:           "Dot",
+	Unknown:       "Unknown",
+}
+
+func (k Kind) String() string {
+	if s, ok := kindNames[k]; ok {
+		return s
+	}
+	return "Kind(?)"
+}
+
+// Token is a single lexical unit covering Text == src[Off:Off+len(Text)].
+type Token struct {
+	Kind Kind
+	Text string
+	Off  int // byte offset of the first byte
+	Line int // 1-based line of the first byte
+	Col  int // 1-based byte column of the first byte
+}
+
+// IsTrivia reports whether the token is whitespace or a comment.
+func (t Token) IsTrivia() bool {
+	switch t.Kind {
+	case Whitespace, LineComment, BlockComment:
+		return true
+	}
+	return false
+}