fix(tokenizer): restore default dialect on pool return

CHANGELOG.md

@@ -7,6 +7,18 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+### Fixed
+- **Tokenizer pool no longer leaks dialect state**: a tokenizer whose dialect was
+  changed via `SetDialect` (e.g. by `validate --dialect mysql`) is now restored to
+  the default (PostgreSQL) dialect when returned to the pool via `PutTokenizer`.
+  Previously `Reset` did not clear the dialect/keywords, so a later
+  `GetTokenizer` caller could receive a tokenizer configured for the wrong
+  dialect and silently mis-parse. The restore lives in `PutTokenizer` (the pool
+  boundary), not `Reset`, because `Reset` also runs at the start of every
+  `Tokenize` call where the configured dialect must persist. The keyword table is
+  rebuilt only when a non-default dialect was configured, so the common path is
+  unaffected.
+
 ## [1.14.0] - 2026-04-12 — Dialect-Aware Transforms, Snowflake 100%, Schema Introspection
 
 Headline themes: dialect-aware transforms, Snowflake at 100% of the QA corpus, ClickHouse significantly expanded (83% of the QA corpus, up from 53%), live schema introspection, SQL transpilation, and first-class integration sub-modules (OpenTelemetry and GORM). Drop-in upgrade from v1.13.0 — no breaking changes.

pkg/sql/tokenizer/pool.go

@@ -19,6 +19,7 @@ import (
 	"sync"
 
 	"github.com/ajitpratap0/GoSQLX/pkg/metrics"
+	"github.com/ajitpratap0/GoSQLX/pkg/sql/keywords"
 )
 
 // bufferPool is used to reuse bytes.Buffer instances during tokenization.
@@ -125,10 +126,22 @@ func GetTokenizer() *Tokenizer {
 //   - Position tracking reset to initial state
 //   - Line tracking cleared but capacity preserved
 //   - Debug logger cleared
-//   - Keywords preserved (immutable configuration)
+//   - Dialect restored to the default (PostgreSQL) so dialect-specific keyword
+//     state does not leak to the next caller
 func PutTokenizer(t *Tokenizer) {
 	if t != nil {
 		t.Reset()
+
+		// Restore the default dialect so a pooled tokenizer never leaks
+		// dialect-specific keyword state to a later caller. This is done here
+		// (at the pool boundary) rather than in Reset, which also runs at the
+		// start of every Tokenize call where the configured dialect must
+		// persist. SetDialect rebuilds the keyword table, so only pay that cost
+		// when a non-default dialect was configured.
+		if t.dialect != keywords.DialectPostgreSQL {
+			t.SetDialect(keywords.DialectPostgreSQL)
+		}
+
 		tokenizerPool.Put(t)
 
 		// Record pool return
@@ -153,7 +166,9 @@ func PutTokenizer(t *Tokenizer) {
 //   - pos: Line 1, Column 0, Index 0
 //   - lineStarts: Empty slice with preserved capacity (contains [0])
 //   - input: nil (ready for new input)
-//   - keywords: Preserved (immutable, no need to reset)
+//   - dialect/keywords: preserved (Reset also runs at the start of every
+//     Tokenize call, so the configured dialect must persist; the pool restores
+//     the default dialect in PutTokenizer instead)
 //   - logger: nil (must be set again if needed)
 //
 // Performance: By preserving slice capacity, subsequent Tokenize() calls
@@ -178,7 +193,9 @@ func (t *Tokenizer) Reset() {
 
 	t.line = 0
 
-	// Don't reset keywords as they're constant
+	// Don't reset keywords/dialect here: Reset also runs at the start of every
+	// Tokenize call, where the configured dialect must persist across calls.
+	// Dialect is restored to the default in PutTokenizer, at the pool boundary.
 	t.logger = nil
 
 	// Preserve Comments slice capacity but reset length

pkg/sql/tokenizer/pool_dialect_test.go

@@ -0,0 +1,92 @@
+// Copyright 2026 GoSQLX Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package tokenizer - pool_dialect_test.go
+// Regression tests ensuring the tokenizer pool does not leak dialect-specific
+// keyword state between callers.
+
+package tokenizer
+
+import (
+	"testing"
+
+	"github.com/ajitpratap0/GoSQLX/pkg/sql/keywords"
+)
+
+// TestReset_PreservesDialect verifies that Reset preserves the configured
+// dialect. Reset runs at the start of every Tokenize call, so a dialect set via
+// SetDialect must survive it — otherwise "set dialect once, tokenize many"
+// would break.
+func TestReset_PreservesDialect(t *testing.T) {
+	tkz, err := New()
+	if err != nil {
+		t.Fatalf("New() error = %v", err)
+	}
+
+	tkz.SetDialect(keywords.DialectMySQL)
+	tkz.Reset()
+	if tkz.Dialect() != keywords.DialectMySQL {
+		t.Errorf("after Reset, dialect = %q, want %q (dialect must persist across Tokenize)",
+			tkz.Dialect(), keywords.DialectMySQL)
+	}
+}
+
+// TestTokenize_PreservesDialectAcrossCalls guards the "set dialect once,
+// tokenize many" pattern end-to-end: the dialect must survive the internal
+// Reset that Tokenize performs on each call.
+func TestTokenize_PreservesDialectAcrossCalls(t *testing.T) {
+	tkz, err := NewWithDialect(keywords.DialectSQLServer)
+	if err != nil {
+		t.Fatalf("NewWithDialect error = %v", err)
+	}
+	for i := 0; i < 3; i++ {
+		if _, err := tkz.Tokenize([]byte("SELECT @@VERSION")); err != nil {
+			t.Fatalf("Tokenize #%d error = %v", i, err)
+		}
+		if tkz.Dialect() != keywords.DialectSQLServer {
+			t.Fatalf("after Tokenize #%d, dialect = %q, want %q",
+				i, tkz.Dialect(), keywords.DialectSQLServer)
+		}
+	}
+}
+
+// TestPool_NoDialectLeak verifies that returning a dialect-configured tokenizer
+// to the pool does not leak that dialect to a subsequent acquisition.
+func TestPool_NoDialectLeak(t *testing.T) {
+	// Acquire, switch to a non-default dialect, and return to the pool.
+	a := GetTokenizer()
+	a.SetDialect(keywords.DialectMySQL)
+	PutTokenizer(a)
+
+	// The next acquisition must be a clean PostgreSQL-default tokenizer.
+	b := GetTokenizer()
+	defer PutTokenizer(b)
+	if b.Dialect() != keywords.DialectPostgreSQL {
+		t.Errorf("pooled tokenizer leaked dialect: got %q, want %q",
+			b.Dialect(), keywords.DialectPostgreSQL)
+	}
+}
+
+// TestReset_DefaultDialectStaysDefault is a sanity check that the common path
+// (already the default dialect) still ends up at the default after Reset.
+func TestReset_DefaultDialectStaysDefault(t *testing.T) {
+	tkz, err := New()
+	if err != nil {
+		t.Fatalf("New() error = %v", err)
+	}
+	tkz.Reset()
+	if tkz.Dialect() != keywords.DialectPostgreSQL {
+		t.Errorf("after Reset, dialect = %q, want %q", tkz.Dialect(), keywords.DialectPostgreSQL)
+	}
+}