147 lines
3.5 KiB
Go
147 lines
3.5 KiB
Go
package lexer
|
|
|
|
import (
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
"testing"
|
|
)
|
|
|
|
// emit reconstructs the source from a token stream.
|
|
func emit(toks []Token) string {
|
|
var b strings.Builder
|
|
for _, t := range toks {
|
|
b.WriteString(t.Text)
|
|
}
|
|
return b.String()
|
|
}
|
|
|
|
func TestRoundTripSmall(t *testing.T) {
|
|
cases := []string{
|
|
"",
|
|
"SELECT 1;",
|
|
"select * from t where a = b;",
|
|
"-- a comment\nSELECT 1",
|
|
"/* block /* nested */ still */ SELECT 1",
|
|
"SELECT 'it''s', E'a\\nb', $$dollar$$, $tag$x$tag$, $1;",
|
|
"a->>'b'::text",
|
|
"x := y + 1;",
|
|
"a=-b",
|
|
"SELECT 1.5, .5, 1e10, 0xFF, 1_000;",
|
|
"arr[1:2]",
|
|
"\"Quoted Ident\".col",
|
|
"U&'d\\0061t'",
|
|
}
|
|
for _, src := range cases {
|
|
got := emit(Lex(src))
|
|
if got != src {
|
|
t.Errorf("round-trip mismatch\n in: %q\nout: %q", src, got)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestKinds(t *testing.T) {
|
|
toks := nonEOF(Lex("a->>'b'::text"))
|
|
want := []Kind{Ident, Operator, String, Operator, Ident}
|
|
if len(toks) != len(want) {
|
|
t.Fatalf("got %d tokens, want %d: %v", len(toks), len(want), toks)
|
|
}
|
|
for i, k := range want {
|
|
if toks[i].Kind != k {
|
|
t.Errorf("token %d: got %s, want %s (text %q)", i, toks[i].Kind, k, toks[i].Text)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestOperatorTrailingRule(t *testing.T) {
|
|
// "=-" must split into "=" and "-" (no special char, can't end in -).
|
|
toks := nonEOF(Lex("a=-b"))
|
|
if len(toks) != 4 || toks[1].Text != "=" || toks[2].Text != "-" {
|
|
t.Fatalf("a=-b mis-lexed: %v", toks)
|
|
}
|
|
// "@-" keeps trailing - because @ is special.
|
|
toks = nonEOF(Lex("a@-b"))
|
|
if toks[1].Text != "@-" {
|
|
t.Fatalf("@- should stay one operator, got %q", toks[1].Text)
|
|
}
|
|
}
|
|
|
|
func TestDollarQuote(t *testing.T) {
|
|
toks := nonEOF(Lex("$func$ body $$ inner $func$"))
|
|
if len(toks) != 1 || toks[0].Kind != DollarString {
|
|
t.Fatalf("dollar quote not single token: %v", toks)
|
|
}
|
|
}
|
|
|
|
func TestLineColumns(t *testing.T) {
|
|
toks := nonEOF(Lex("ab\n cd"))
|
|
// ab(1,1) ws cd(2,3)
|
|
if toks[0].Line != 1 || toks[0].Col != 1 {
|
|
t.Errorf("ab at %d:%d, want 1:1", toks[0].Line, toks[0].Col)
|
|
}
|
|
cd := toks[len(toks)-1]
|
|
if cd.Text != "cd" || cd.Line != 2 || cd.Col != 3 {
|
|
t.Errorf("cd at %d:%d, want 2:3", cd.Line, cd.Col)
|
|
}
|
|
}
|
|
|
|
// TestCorpusRoundTrip is the core lossless invariant: lexing then re-emitting
|
|
// every real-world .pgsql fixture must reproduce it byte-for-byte.
|
|
func TestCorpusRoundTrip(t *testing.T) {
|
|
dir := filepath.Join("..", "..", "testdata", "corpus")
|
|
entries, err := os.ReadDir(dir)
|
|
if err != nil {
|
|
t.Skipf("no corpus dir: %v", err)
|
|
}
|
|
var seen int
|
|
for _, e := range entries {
|
|
if e.IsDir() || !strings.HasSuffix(e.Name(), ".pgsql") {
|
|
continue
|
|
}
|
|
seen++
|
|
path := filepath.Join(dir, e.Name())
|
|
data, err := os.ReadFile(path)
|
|
if err != nil {
|
|
t.Fatalf("read %s: %v", path, err)
|
|
}
|
|
src := string(data)
|
|
if got := emit(Lex(src)); got != src {
|
|
t.Errorf("%s: round-trip mismatch (len in=%d out=%d)", e.Name(), len(src), len(got))
|
|
reportFirstDiff(t, e.Name(), src, got)
|
|
}
|
|
}
|
|
if seen == 0 {
|
|
t.Skip("corpus dir has no .pgsql files")
|
|
}
|
|
t.Logf("round-tripped %d corpus files", seen)
|
|
}
|
|
|
|
func reportFirstDiff(t *testing.T, name, a, b string) {
|
|
t.Helper()
|
|
n := len(a)
|
|
if len(b) < n {
|
|
n = len(b)
|
|
}
|
|
for i := 0; i < n; i++ {
|
|
if a[i] != b[i] {
|
|
lo := i - 20
|
|
if lo < 0 {
|
|
lo = 0
|
|
}
|
|
t.Logf("%s: first diff at byte %d\n in: %q\n out: %q", name, i, a[lo:min(i+20, len(a))], b[lo:min(i+20, len(b))])
|
|
return
|
|
}
|
|
}
|
|
}
|
|
|
|
func nonEOF(toks []Token) []Token {
|
|
var out []Token
|
|
for _, t := range toks {
|
|
if t.Kind == EOF {
|
|
continue
|
|
}
|
|
out = append(out, t)
|
|
}
|
|
return out
|
|
}
|