package lexer import ( "os" "path/filepath" "strings" "testing" ) // emit reconstructs the source from a token stream. func emit(toks []Token) string { var b strings.Builder for _, t := range toks { b.WriteString(t.Text) } return b.String() } func TestRoundTripSmall(t *testing.T) { cases := []string{ "", "SELECT 1;", "select * from t where a = b;", "-- a comment\nSELECT 1", "/* block /* nested */ still */ SELECT 1", "SELECT 'it''s', E'a\\nb', $$dollar$$, $tag$x$tag$, $1;", "a->>'b'::text", "x := y + 1;", "a=-b", "SELECT 1.5, .5, 1e10, 0xFF, 1_000;", "arr[1:2]", "\"Quoted Ident\".col", "U&'d\\0061t'", } for _, src := range cases { got := emit(Lex(src)) if got != src { t.Errorf("round-trip mismatch\n in: %q\nout: %q", src, got) } } } func TestKinds(t *testing.T) { toks := nonEOF(Lex("a->>'b'::text")) want := []Kind{Ident, Operator, String, Operator, Ident} if len(toks) != len(want) { t.Fatalf("got %d tokens, want %d: %v", len(toks), len(want), toks) } for i, k := range want { if toks[i].Kind != k { t.Errorf("token %d: got %s, want %s (text %q)", i, toks[i].Kind, k, toks[i].Text) } } } func TestOperatorTrailingRule(t *testing.T) { // "=-" must split into "=" and "-" (no special char, can't end in -). toks := nonEOF(Lex("a=-b")) if len(toks) != 4 || toks[1].Text != "=" || toks[2].Text != "-" { t.Fatalf("a=-b mis-lexed: %v", toks) } // "@-" keeps trailing - because @ is special. toks = nonEOF(Lex("a@-b")) if toks[1].Text != "@-" { t.Fatalf("@- should stay one operator, got %q", toks[1].Text) } } func TestDollarQuote(t *testing.T) { toks := nonEOF(Lex("$func$ body $$ inner $func$")) if len(toks) != 1 || toks[0].Kind != DollarString { t.Fatalf("dollar quote not single token: %v", toks) } } func TestLineColumns(t *testing.T) { toks := nonEOF(Lex("ab\n cd")) // ab(1,1) ws cd(2,3) if toks[0].Line != 1 || toks[0].Col != 1 { t.Errorf("ab at %d:%d, want 1:1", toks[0].Line, toks[0].Col) } cd := toks[len(toks)-1] if cd.Text != "cd" || cd.Line != 2 || cd.Col != 3 { t.Errorf("cd at %d:%d, want 2:3", cd.Line, cd.Col) } } // TestCorpusRoundTrip is the core lossless invariant: lexing then re-emitting // every real-world .pgsql fixture must reproduce it byte-for-byte. func TestCorpusRoundTrip(t *testing.T) { dir := filepath.Join("..", "..", "testdata", "corpus") entries, err := os.ReadDir(dir) if err != nil { t.Skipf("no corpus dir: %v", err) } var seen int for _, e := range entries { if e.IsDir() || !strings.HasSuffix(e.Name(), ".pgsql") { continue } seen++ path := filepath.Join(dir, e.Name()) data, err := os.ReadFile(path) if err != nil { t.Fatalf("read %s: %v", path, err) } src := string(data) if got := emit(Lex(src)); got != src { t.Errorf("%s: round-trip mismatch (len in=%d out=%d)", e.Name(), len(src), len(got)) reportFirstDiff(t, e.Name(), src, got) } } if seen == 0 { t.Skip("corpus dir has no .pgsql files") } t.Logf("round-tripped %d corpus files", seen) } func reportFirstDiff(t *testing.T, name, a, b string) { t.Helper() n := len(a) if len(b) < n { n = len(b) } for i := 0; i < n; i++ { if a[i] != b[i] { lo := i - 20 if lo < 0 { lo = 0 } t.Logf("%s: first diff at byte %d\n in: %q\n out: %q", name, i, a[lo:min(i+20, len(a))], b[lo:min(i+20, len(b))]) return } } } func nonEOF(toks []Token) []Token { var out []Token for _, t := range toks { if t.Kind == EOF { continue } out = append(out, t) } return out }