feat: initial plan

pkg/lexer/lexer_test.go

@@ -0,0 +1,146 @@
+package lexer
+
+import (
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+)
+
+// emit reconstructs the source from a token stream.
+func emit(toks []Token) string {
+	var b strings.Builder
+	for _, t := range toks {
+		b.WriteString(t.Text)
+	}
+	return b.String()
+}
+
+func TestRoundTripSmall(t *testing.T) {
+	cases := []string{
+		"",
+		"SELECT 1;",
+		"select * from t where a = b;",
+		"-- a comment\nSELECT 1",
+		"/* block /* nested */ still */ SELECT 1",
+		"SELECT 'it''s', E'a\\nb', $$dollar$$, $tag$x$tag$, $1;",
+		"a->>'b'::text",
+		"x := y + 1;",
+		"a=-b",
+		"SELECT 1.5, .5, 1e10, 0xFF, 1_000;",
+		"arr[1:2]",
+		"\"Quoted Ident\".col",
+		"U&'d\\0061t'",
+	}
+	for _, src := range cases {
+		got := emit(Lex(src))
+		if got != src {
+			t.Errorf("round-trip mismatch\n in: %q\nout: %q", src, got)
+		}
+	}
+}
+
+func TestKinds(t *testing.T) {
+	toks := nonEOF(Lex("a->>'b'::text"))
+	want := []Kind{Ident, Operator, String, Operator, Ident}
+	if len(toks) != len(want) {
+		t.Fatalf("got %d tokens, want %d: %v", len(toks), len(want), toks)
+	}
+	for i, k := range want {
+		if toks[i].Kind != k {
+			t.Errorf("token %d: got %s, want %s (text %q)", i, toks[i].Kind, k, toks[i].Text)
+		}
+	}
+}
+
+func TestOperatorTrailingRule(t *testing.T) {
+	// "=-" must split into "=" and "-" (no special char, can't end in -).
+	toks := nonEOF(Lex("a=-b"))
+	if len(toks) != 4 || toks[1].Text != "=" || toks[2].Text != "-" {
+		t.Fatalf("a=-b mis-lexed: %v", toks)
+	}
+	// "@-" keeps trailing - because @ is special.
+	toks = nonEOF(Lex("a@-b"))
+	if toks[1].Text != "@-" {
+		t.Fatalf("@- should stay one operator, got %q", toks[1].Text)
+	}
+}
+
+func TestDollarQuote(t *testing.T) {
+	toks := nonEOF(Lex("$func$ body $$ inner $func$"))
+	if len(toks) != 1 || toks[0].Kind != DollarString {
+		t.Fatalf("dollar quote not single token: %v", toks)
+	}
+}
+
+func TestLineColumns(t *testing.T) {
+	toks := nonEOF(Lex("ab\n  cd"))
+	// ab(1,1) ws cd(2,3)
+	if toks[0].Line != 1 || toks[0].Col != 1 {
+		t.Errorf("ab at %d:%d, want 1:1", toks[0].Line, toks[0].Col)
+	}
+	cd := toks[len(toks)-1]
+	if cd.Text != "cd" || cd.Line != 2 || cd.Col != 3 {
+		t.Errorf("cd at %d:%d, want 2:3", cd.Line, cd.Col)
+	}
+}
+
+// TestCorpusRoundTrip is the core lossless invariant: lexing then re-emitting
+// every real-world .pgsql fixture must reproduce it byte-for-byte.
+func TestCorpusRoundTrip(t *testing.T) {
+	dir := filepath.Join("..", "..", "testdata", "corpus")
+	entries, err := os.ReadDir(dir)
+	if err != nil {
+		t.Skipf("no corpus dir: %v", err)
+	}
+	var seen int
+	for _, e := range entries {
+		if e.IsDir() || !strings.HasSuffix(e.Name(), ".pgsql") {
+			continue
+		}
+		seen++
+		path := filepath.Join(dir, e.Name())
+		data, err := os.ReadFile(path)
+		if err != nil {
+			t.Fatalf("read %s: %v", path, err)
+		}
+		src := string(data)
+		if got := emit(Lex(src)); got != src {
+			t.Errorf("%s: round-trip mismatch (len in=%d out=%d)", e.Name(), len(src), len(got))
+			reportFirstDiff(t, e.Name(), src, got)
+		}
+	}
+	if seen == 0 {
+		t.Skip("corpus dir has no .pgsql files")
+	}
+	t.Logf("round-tripped %d corpus files", seen)
+}
+
+func reportFirstDiff(t *testing.T, name, a, b string) {
+	t.Helper()
+	n := len(a)
+	if len(b) < n {
+		n = len(b)
+	}
+	for i := 0; i < n; i++ {
+		if a[i] != b[i] {
+			lo := i - 20
+			if lo < 0 {
+				lo = 0
+			}
+			t.Logf("%s: first diff at byte %d\n  in:  %q\n  out: %q", name, i, a[lo:min(i+20, len(a))], b[lo:min(i+20, len(b))])
+			return
+		}
+	}
+}
+
+func nonEOF(toks []Token) []Token {
+	var out []Token
+	for _, t := range toks {
+		if t.Kind == EOF {
+			continue
+		}
+		out = append(out, t)
+	}
+	return out
+}