feat(tokenizer): support ? placeholder for MySQL/MariaDB/SQLite

CHANGELOG.md

@@ -7,6 +7,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+### Added
+- **MySQL/MariaDB/SQLite `?` positional parameter placeholders**: a bare `?` is
+  now tokenized as a parameter placeholder (`TokenTypePlaceholder`) when parsing
+  with `DialectMySQL`, `DialectMariaDB`, or `DialectSQLite` (e.g.
+  `SELECT * FROM users WHERE id = ?`). This matches the placeholder style used by
+  the standard MySQL and SQLite `database/sql` drivers. Dialect-gated: PostgreSQL
+  (the default dialect) continues to treat `?`, `?|`, and `?&` as JSON
+  key-existence operators.
+
 ## [1.14.0] - 2026-04-12 — Dialect-Aware Transforms, Snowflake 100%, Schema Introspection
 
 Headline themes: dialect-aware transforms, Snowflake at 100% of the QA corpus, ClickHouse significantly expanded (83% of the QA corpus, up from 53%), live schema introspection, SQL transpilation, and first-class integration sub-modules (OpenTelemetry and GORM). Drop-in upgrade from v1.13.0 — no breaking changes.

docs/SQL_COMPATIBILITY.md

@@ -41,7 +41,7 @@ This matrix documents the comprehensive SQL feature support in GoSQLX across dif
 - ✅ **PostgreSQL Enhancements**:
   - **Type Casting** - `::` operator for PostgreSQL-style casts (`SELECT 1::int`)
   - **UPSERT** - `INSERT ... ON CONFLICT DO UPDATE/NOTHING`
-  - **Positional Parameters** - `$1`, `$2` style parameter placeholders
+  - **Positional Parameters** - `$1`, `$2` style parameter placeholders (PostgreSQL); `?` placeholders for MySQL/MariaDB/SQLite (dialect-gated)
   - **JSONB Operators** - Additional `@?` and `@@` operators
   - **Regex Operators** - `~`, `~*`, `!~`, `!~*` for pattern matching
 - ✅ **ARRAY Constructors**: `ARRAY[1, 2, 3]` expressions with subscript/slice operations

pkg/gosqlx/parse_with_dialect_test.go

@@ -66,6 +66,26 @@ func TestParseWithDialect_MySQLOnDuplicateKeyUpdate(t *testing.T) {
 	}
 }
 
+func TestParseWithDialect_QuestionMarkPlaceholder(t *testing.T) {
+	// MySQL, MariaDB, and SQLite use a bare ? as a positional bind parameter.
+	for _, dialect := range []keywords.SQLDialect{
+		keywords.DialectMySQL,
+		keywords.DialectMariaDB,
+		keywords.DialectSQLite,
+	} {
+		t.Run(string(dialect), func(t *testing.T) {
+			sql := "SELECT * FROM users WHERE id = ? AND status = ?"
+			tree, err := gosqlx.ParseWithDialect(sql, dialect)
+			if err != nil {
+				t.Fatalf("ParseWithDialect(%s) unexpected error: %v", dialect, err)
+			}
+			if len(tree.Statements) != 1 {
+				t.Fatalf("expected 1 statement, got %d", len(tree.Statements))
+			}
+		})
+	}
+}
+
 func TestParseWithDialect_UnknownDialectReturnsError(t *testing.T) {
 	_, err := gosqlx.ParseWithDialect("SELECT 1", "totally-unknown-dialect")
 	if err == nil {

pkg/sql/parser/expressions_literal.go

@@ -255,7 +255,8 @@ func (p *Parser) parsePrimaryExpression() (ast.Expression, error) {
 	}
 
 	if p.isType(models.TokenTypePlaceholder) {
-		// Handle SQL placeholders (e.g., $1, $2 for PostgreSQL; @param for SQL Server)
+		// Handle SQL placeholders (e.g., $1, $2 for PostgreSQL; @param for
+		// SQL Server; ? for MySQL/MariaDB/SQLite)
 		value := p.currentToken.Token.Value
 		p.advance()
 		return &ast.LiteralValue{Value: value, Type: "placeholder"}, nil

pkg/sql/parser/question_placeholder_test.go

@@ -0,0 +1,63 @@
+// Copyright 2026 GoSQLX Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package parser
+
+import (
+	"testing"
+
+	"github.com/ajitpratap0/GoSQLX/pkg/sql/ast"
+	"github.com/ajitpratap0/GoSQLX/pkg/sql/keywords"
+)
+
+// TestParseWithDialect_QuestionMarkPlaceholder verifies that a bare `?` parses
+// as a placeholder literal end-to-end for MySQL, MariaDB, and SQLite.
+func TestParseWithDialect_QuestionMarkPlaceholder(t *testing.T) {
+	dialects := []keywords.SQLDialect{
+		keywords.DialectMySQL,
+		keywords.DialectMariaDB,
+		keywords.DialectSQLite,
+	}
+
+	for _, dialect := range dialects {
+		t.Run(string(dialect), func(t *testing.T) {
+			tree, err := ParseWithDialect("SELECT * FROM users WHERE id = ?", dialect)
+			if err != nil {
+				t.Fatalf("ParseWithDialect(%q) error = %v", dialect, err)
+			}
+			defer ast.ReleaseAST(tree)
+
+			stmt, ok := tree.Statements[0].(*ast.SelectStatement)
+			if !ok {
+				t.Fatalf("expected *ast.SelectStatement, got %T", tree.Statements[0])
+			}
+
+			where, ok := stmt.Where.(*ast.BinaryExpression)
+			if !ok {
+				t.Fatalf("expected WHERE to be *ast.BinaryExpression, got %T", stmt.Where)
+			}
+
+			lit, ok := where.Right.(*ast.LiteralValue)
+			if !ok {
+				t.Fatalf("expected right side to be *ast.LiteralValue, got %T", where.Right)
+			}
+			if lit.Type != "placeholder" {
+				t.Errorf("expected placeholder type, got %q", lit.Type)
+			}
+			if lit.Value != "?" {
+				t.Errorf("expected placeholder value %q, got %q", "?", lit.Value)
+			}
+		})
+	}
+}

pkg/sql/tokenizer/question_param_test.go

@@ -0,0 +1,182 @@
+// Copyright 2026 GoSQLX Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package tokenizer - question_param_test.go
+// Tests for MySQL/MariaDB/SQLite `?` positional parameter placeholder
+// tokenization, and the regression guard that PostgreSQL keeps `?` as its
+// JSON key-existence operator.
+
+package tokenizer
+
+import (
+	"testing"
+
+	"github.com/ajitpratap0/GoSQLX/pkg/models"
+	"github.com/ajitpratap0/GoSQLX/pkg/sql/keywords"
+)
+
+func TestTokenizer_QuestionMarkPlaceholder(t *testing.T) {
+	dialects := []keywords.SQLDialect{
+		keywords.DialectMySQL,
+		keywords.DialectMariaDB,
+		keywords.DialectSQLite,
+	}
+
+	tests := []struct {
+		name     string
+		input    string
+		expected []struct {
+			tokenType models.TokenType
+			value     string
+		}
+	}{
+		{
+			name:  "Single ? placeholder",
+			input: "SELECT * FROM users WHERE id = ?",
+			expected: []struct {
+				tokenType models.TokenType
+				value     string
+			}{
+				{models.TokenTypeSelect, "SELECT"},
+				{models.TokenTypeMul, "*"},
+				{models.TokenTypeFrom, "FROM"},
+				{models.TokenTypeIdentifier, "users"},
+				{models.TokenTypeWhere, "WHERE"},
+				{models.TokenTypeIdentifier, "id"},
+				{models.TokenTypeEq, "="},
+				{models.TokenTypePlaceholder, "?"},
+			},
+		},
+		{
+			name:  "Multiple ? placeholders in VALUES",
+			input: "INSERT INTO users (name, email) VALUES (?, ?, ?)",
+			expected: []struct {
+				tokenType models.TokenType
+				value     string
+			}{
+				{models.TokenTypeInsert, "INSERT"},
+				{models.TokenTypeInto, "INTO"},
+				{models.TokenTypeIdentifier, "users"},
+				{models.TokenTypeLParen, "("},
+				{models.TokenTypeIdentifier, "name"},
+				{models.TokenTypeComma, ","},
+				{models.TokenTypeIdentifier, "email"},
+				{models.TokenTypeRParen, ")"},
+				{models.TokenTypeValues, "VALUES"},
+				{models.TokenTypeLParen, "("},
+				{models.TokenTypePlaceholder, "?"},
+				{models.TokenTypeComma, ","},
+				{models.TokenTypePlaceholder, "?"},
+				{models.TokenTypeComma, ","},
+				{models.TokenTypePlaceholder, "?"},
+				{models.TokenTypeRParen, ")"},
+			},
+		},
+		{
+			name:  "? placeholder without surrounding space",
+			input: "SELECT name FROM users WHERE id=?",
+			expected: []struct {
+				tokenType models.TokenType
+				value     string
+			}{
+				{models.TokenTypeSelect, "SELECT"},
+				{models.TokenTypeIdentifier, "name"},
+				{models.TokenTypeFrom, "FROM"},
+				{models.TokenTypeIdentifier, "users"},
+				{models.TokenTypeWhere, "WHERE"},
+				{models.TokenTypeIdentifier, "id"},
+				{models.TokenTypeEq, "="},
+				{models.TokenTypePlaceholder, "?"},
+			},
+		},
+	}
+
+	for _, dialect := range dialects {
+		for _, tt := range tests {
+			t.Run(string(dialect)+"/"+tt.name, func(t *testing.T) {
+				tkz, err := NewWithDialect(dialect)
+				if err != nil {
+					t.Fatalf("NewWithDialect(%q) error = %v", dialect, err)
+				}
+
+				tokens, err := tkz.Tokenize([]byte(tt.input))
+				if err != nil {
+					t.Fatalf("Tokenize() error = %v", err)
+				}
+
+				// Remove EOF token
+				tokens = tokens[:len(tokens)-1]
+
+				if len(tokens) != len(tt.expected) {
+					t.Fatalf("Expected %d tokens, got %d", len(tt.expected), len(tokens))
+				}
+
+				for i, exp := range tt.expected {
+					if tokens[i].Token.Type != exp.tokenType {
+						t.Errorf("Token %d: expected type %s, got %s (value: %s)",
+							i, exp.tokenType.String(), tokens[i].Token.Type.String(), tokens[i].Token.Value)
+					}
+					if tokens[i].Token.Value != exp.value {
+						t.Errorf("Token %d: expected value %q, got %q",
+							i, exp.value, tokens[i].Token.Value)
+					}
+				}
+			})
+		}
+	}
+}
+
+// TestTokenizer_QuestionMarkPostgreSQLJSONOperator guards against a regression
+// where `?`, `?|`, and `?&` under PostgreSQL would be mistaken for placeholders
+// instead of JSON operators.
+func TestTokenizer_QuestionMarkPostgreSQLJSONOperator(t *testing.T) {
+	tests := []struct {
+		name      string
+		input     string
+		tokenType models.TokenType
+		value     string
+	}{
+		{"single ? existence operator", "data ? 'key'", models.TokenTypeQuestion, "?"},
+		{"?| any key exists", "data ?| array['a','b']", models.TokenTypeQuestionPipe, "?|"},
+		{"?& all keys exist", "data ?& array['a','b']", models.TokenTypeQuestionAnd, "?&"},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			tkz, err := NewWithDialect(keywords.DialectPostgreSQL)
+			if err != nil {
+				t.Fatalf("NewWithDialect() error = %v", err)
+			}
+
+			tokens, err := tkz.Tokenize([]byte(tt.input))
+			if err != nil {
+				t.Fatalf("Tokenize() error = %v", err)
+			}
+
+			var found bool
+			for _, tok := range tokens {
+				if tok.Token.Type == tt.tokenType && tok.Token.Value == tt.value {
+					found = true
+					break
+				}
+				if tok.Token.Type == models.TokenTypePlaceholder {
+					t.Fatalf("PostgreSQL emitted a placeholder for %q; expected JSON operator", tt.input)
+				}
+			}
+			if !found {
+				t.Errorf("expected token %s %q in %q", tt.tokenType.String(), tt.value, tt.input)
+			}
+		})
+	}
+}

pkg/sql/tokenizer/tokenizer.go

@@ -1626,6 +1626,15 @@ func (t *Tokenizer) readPunctuation() (models.Token, error) {
 		return models.Token{Type: models.TokenTypeSharp, Value: "#"}, nil
 	case '?':
 		t.pos.AdvanceRune(r, size)
+		// MySQL, MariaDB, and SQLite use a bare ? as a positional parameter
+		// placeholder (e.g. WHERE id = ?). These dialects have no JSON ?
+		// existence operator, so emit a placeholder token before the JSON
+		// operator lookahead below.
+		if t.dialect == keywords.DialectMySQL ||
+			t.dialect == keywords.DialectMariaDB ||
+			t.dialect == keywords.DialectSQLite {
+			return models.Token{Type: models.TokenTypePlaceholder, Value: "?"}, nil
+		}
 		// Check for PostgreSQL JSON operators
 		if t.pos.Index < len(t.input) {
 			nextR, nextSize := utf8.DecodeRune(t.input[t.pos.Index:])