diff --git a/CHANGELOG.md b/CHANGELOG.md index 5810b785..df861251 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added +- **MySQL/MariaDB/SQLite `?` positional parameter placeholders**: a bare `?` is + now tokenized as a parameter placeholder (`TokenTypePlaceholder`) when parsing + with `DialectMySQL`, `DialectMariaDB`, or `DialectSQLite` (e.g. + `SELECT * FROM users WHERE id = ?`). This matches the placeholder style used by + the standard MySQL and SQLite `database/sql` drivers. Dialect-gated: PostgreSQL + (the default dialect) continues to treat `?`, `?|`, and `?&` as JSON + key-existence operators. + ## [1.14.0] - 2026-04-12 — Dialect-Aware Transforms, Snowflake 100%, Schema Introspection Headline themes: dialect-aware transforms, Snowflake at 100% of the QA corpus, ClickHouse significantly expanded (83% of the QA corpus, up from 53%), live schema introspection, SQL transpilation, and first-class integration sub-modules (OpenTelemetry and GORM). Drop-in upgrade from v1.13.0 — no breaking changes. diff --git a/docs/SQL_COMPATIBILITY.md b/docs/SQL_COMPATIBILITY.md index 2e2abdbe..963fb38b 100644 --- a/docs/SQL_COMPATIBILITY.md +++ b/docs/SQL_COMPATIBILITY.md @@ -41,7 +41,7 @@ This matrix documents the comprehensive SQL feature support in GoSQLX across dif - ✅ **PostgreSQL Enhancements**: - **Type Casting** - `::` operator for PostgreSQL-style casts (`SELECT 1::int`) - **UPSERT** - `INSERT ... ON CONFLICT DO UPDATE/NOTHING` - - **Positional Parameters** - `$1`, `$2` style parameter placeholders + - **Positional Parameters** - `$1`, `$2` style parameter placeholders (PostgreSQL); `?` placeholders for MySQL/MariaDB/SQLite (dialect-gated) - **JSONB Operators** - Additional `@?` and `@@` operators - **Regex Operators** - `~`, `~*`, `!~`, `!~*` for pattern matching - ✅ **ARRAY Constructors**: `ARRAY[1, 2, 3]` expressions with subscript/slice operations diff --git a/pkg/gosqlx/parse_with_dialect_test.go b/pkg/gosqlx/parse_with_dialect_test.go index a4f7d396..d70a9d0a 100644 --- a/pkg/gosqlx/parse_with_dialect_test.go +++ b/pkg/gosqlx/parse_with_dialect_test.go @@ -66,6 +66,26 @@ func TestParseWithDialect_MySQLOnDuplicateKeyUpdate(t *testing.T) { } } +func TestParseWithDialect_QuestionMarkPlaceholder(t *testing.T) { + // MySQL, MariaDB, and SQLite use a bare ? as a positional bind parameter. + for _, dialect := range []keywords.SQLDialect{ + keywords.DialectMySQL, + keywords.DialectMariaDB, + keywords.DialectSQLite, + } { + t.Run(string(dialect), func(t *testing.T) { + sql := "SELECT * FROM users WHERE id = ? AND status = ?" + tree, err := gosqlx.ParseWithDialect(sql, dialect) + if err != nil { + t.Fatalf("ParseWithDialect(%s) unexpected error: %v", dialect, err) + } + if len(tree.Statements) != 1 { + t.Fatalf("expected 1 statement, got %d", len(tree.Statements)) + } + }) + } +} + func TestParseWithDialect_UnknownDialectReturnsError(t *testing.T) { _, err := gosqlx.ParseWithDialect("SELECT 1", "totally-unknown-dialect") if err == nil { diff --git a/pkg/sql/parser/expressions_literal.go b/pkg/sql/parser/expressions_literal.go index 04516108..b9396ea5 100644 --- a/pkg/sql/parser/expressions_literal.go +++ b/pkg/sql/parser/expressions_literal.go @@ -255,7 +255,8 @@ func (p *Parser) parsePrimaryExpression() (ast.Expression, error) { } if p.isType(models.TokenTypePlaceholder) { - // Handle SQL placeholders (e.g., $1, $2 for PostgreSQL; @param for SQL Server) + // Handle SQL placeholders (e.g., $1, $2 for PostgreSQL; @param for + // SQL Server; ? for MySQL/MariaDB/SQLite) value := p.currentToken.Token.Value p.advance() return &ast.LiteralValue{Value: value, Type: "placeholder"}, nil diff --git a/pkg/sql/parser/question_placeholder_test.go b/pkg/sql/parser/question_placeholder_test.go new file mode 100644 index 00000000..8e49c27a --- /dev/null +++ b/pkg/sql/parser/question_placeholder_test.go @@ -0,0 +1,63 @@ +// Copyright 2026 GoSQLX Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package parser + +import ( + "testing" + + "github.com/ajitpratap0/GoSQLX/pkg/sql/ast" + "github.com/ajitpratap0/GoSQLX/pkg/sql/keywords" +) + +// TestParseWithDialect_QuestionMarkPlaceholder verifies that a bare `?` parses +// as a placeholder literal end-to-end for MySQL, MariaDB, and SQLite. +func TestParseWithDialect_QuestionMarkPlaceholder(t *testing.T) { + dialects := []keywords.SQLDialect{ + keywords.DialectMySQL, + keywords.DialectMariaDB, + keywords.DialectSQLite, + } + + for _, dialect := range dialects { + t.Run(string(dialect), func(t *testing.T) { + tree, err := ParseWithDialect("SELECT * FROM users WHERE id = ?", dialect) + if err != nil { + t.Fatalf("ParseWithDialect(%q) error = %v", dialect, err) + } + defer ast.ReleaseAST(tree) + + stmt, ok := tree.Statements[0].(*ast.SelectStatement) + if !ok { + t.Fatalf("expected *ast.SelectStatement, got %T", tree.Statements[0]) + } + + where, ok := stmt.Where.(*ast.BinaryExpression) + if !ok { + t.Fatalf("expected WHERE to be *ast.BinaryExpression, got %T", stmt.Where) + } + + lit, ok := where.Right.(*ast.LiteralValue) + if !ok { + t.Fatalf("expected right side to be *ast.LiteralValue, got %T", where.Right) + } + if lit.Type != "placeholder" { + t.Errorf("expected placeholder type, got %q", lit.Type) + } + if lit.Value != "?" { + t.Errorf("expected placeholder value %q, got %q", "?", lit.Value) + } + }) + } +} diff --git a/pkg/sql/tokenizer/question_param_test.go b/pkg/sql/tokenizer/question_param_test.go new file mode 100644 index 00000000..13d962cb --- /dev/null +++ b/pkg/sql/tokenizer/question_param_test.go @@ -0,0 +1,182 @@ +// Copyright 2026 GoSQLX Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package tokenizer - question_param_test.go +// Tests for MySQL/MariaDB/SQLite `?` positional parameter placeholder +// tokenization, and the regression guard that PostgreSQL keeps `?` as its +// JSON key-existence operator. + +package tokenizer + +import ( + "testing" + + "github.com/ajitpratap0/GoSQLX/pkg/models" + "github.com/ajitpratap0/GoSQLX/pkg/sql/keywords" +) + +func TestTokenizer_QuestionMarkPlaceholder(t *testing.T) { + dialects := []keywords.SQLDialect{ + keywords.DialectMySQL, + keywords.DialectMariaDB, + keywords.DialectSQLite, + } + + tests := []struct { + name string + input string + expected []struct { + tokenType models.TokenType + value string + } + }{ + { + name: "Single ? placeholder", + input: "SELECT * FROM users WHERE id = ?", + expected: []struct { + tokenType models.TokenType + value string + }{ + {models.TokenTypeSelect, "SELECT"}, + {models.TokenTypeMul, "*"}, + {models.TokenTypeFrom, "FROM"}, + {models.TokenTypeIdentifier, "users"}, + {models.TokenTypeWhere, "WHERE"}, + {models.TokenTypeIdentifier, "id"}, + {models.TokenTypeEq, "="}, + {models.TokenTypePlaceholder, "?"}, + }, + }, + { + name: "Multiple ? placeholders in VALUES", + input: "INSERT INTO users (name, email) VALUES (?, ?, ?)", + expected: []struct { + tokenType models.TokenType + value string + }{ + {models.TokenTypeInsert, "INSERT"}, + {models.TokenTypeInto, "INTO"}, + {models.TokenTypeIdentifier, "users"}, + {models.TokenTypeLParen, "("}, + {models.TokenTypeIdentifier, "name"}, + {models.TokenTypeComma, ","}, + {models.TokenTypeIdentifier, "email"}, + {models.TokenTypeRParen, ")"}, + {models.TokenTypeValues, "VALUES"}, + {models.TokenTypeLParen, "("}, + {models.TokenTypePlaceholder, "?"}, + {models.TokenTypeComma, ","}, + {models.TokenTypePlaceholder, "?"}, + {models.TokenTypeComma, ","}, + {models.TokenTypePlaceholder, "?"}, + {models.TokenTypeRParen, ")"}, + }, + }, + { + name: "? placeholder without surrounding space", + input: "SELECT name FROM users WHERE id=?", + expected: []struct { + tokenType models.TokenType + value string + }{ + {models.TokenTypeSelect, "SELECT"}, + {models.TokenTypeIdentifier, "name"}, + {models.TokenTypeFrom, "FROM"}, + {models.TokenTypeIdentifier, "users"}, + {models.TokenTypeWhere, "WHERE"}, + {models.TokenTypeIdentifier, "id"}, + {models.TokenTypeEq, "="}, + {models.TokenTypePlaceholder, "?"}, + }, + }, + } + + for _, dialect := range dialects { + for _, tt := range tests { + t.Run(string(dialect)+"/"+tt.name, func(t *testing.T) { + tkz, err := NewWithDialect(dialect) + if err != nil { + t.Fatalf("NewWithDialect(%q) error = %v", dialect, err) + } + + tokens, err := tkz.Tokenize([]byte(tt.input)) + if err != nil { + t.Fatalf("Tokenize() error = %v", err) + } + + // Remove EOF token + tokens = tokens[:len(tokens)-1] + + if len(tokens) != len(tt.expected) { + t.Fatalf("Expected %d tokens, got %d", len(tt.expected), len(tokens)) + } + + for i, exp := range tt.expected { + if tokens[i].Token.Type != exp.tokenType { + t.Errorf("Token %d: expected type %s, got %s (value: %s)", + i, exp.tokenType.String(), tokens[i].Token.Type.String(), tokens[i].Token.Value) + } + if tokens[i].Token.Value != exp.value { + t.Errorf("Token %d: expected value %q, got %q", + i, exp.value, tokens[i].Token.Value) + } + } + }) + } + } +} + +// TestTokenizer_QuestionMarkPostgreSQLJSONOperator guards against a regression +// where `?`, `?|`, and `?&` under PostgreSQL would be mistaken for placeholders +// instead of JSON operators. +func TestTokenizer_QuestionMarkPostgreSQLJSONOperator(t *testing.T) { + tests := []struct { + name string + input string + tokenType models.TokenType + value string + }{ + {"single ? existence operator", "data ? 'key'", models.TokenTypeQuestion, "?"}, + {"?| any key exists", "data ?| array['a','b']", models.TokenTypeQuestionPipe, "?|"}, + {"?& all keys exist", "data ?& array['a','b']", models.TokenTypeQuestionAnd, "?&"}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + tkz, err := NewWithDialect(keywords.DialectPostgreSQL) + if err != nil { + t.Fatalf("NewWithDialect() error = %v", err) + } + + tokens, err := tkz.Tokenize([]byte(tt.input)) + if err != nil { + t.Fatalf("Tokenize() error = %v", err) + } + + var found bool + for _, tok := range tokens { + if tok.Token.Type == tt.tokenType && tok.Token.Value == tt.value { + found = true + break + } + if tok.Token.Type == models.TokenTypePlaceholder { + t.Fatalf("PostgreSQL emitted a placeholder for %q; expected JSON operator", tt.input) + } + } + if !found { + t.Errorf("expected token %s %q in %q", tt.tokenType.String(), tt.value, tt.input) + } + }) + } +} diff --git a/pkg/sql/tokenizer/tokenizer.go b/pkg/sql/tokenizer/tokenizer.go index f7a74e5a..3ed3ec01 100644 --- a/pkg/sql/tokenizer/tokenizer.go +++ b/pkg/sql/tokenizer/tokenizer.go @@ -1626,6 +1626,15 @@ func (t *Tokenizer) readPunctuation() (models.Token, error) { return models.Token{Type: models.TokenTypeSharp, Value: "#"}, nil case '?': t.pos.AdvanceRune(r, size) + // MySQL, MariaDB, and SQLite use a bare ? as a positional parameter + // placeholder (e.g. WHERE id = ?). These dialects have no JSON ? + // existence operator, so emit a placeholder token before the JSON + // operator lookahead below. + if t.dialect == keywords.DialectMySQL || + t.dialect == keywords.DialectMariaDB || + t.dialect == keywords.DialectSQLite { + return models.Token{Type: models.TokenTypePlaceholder, Value: "?"}, nil + } // Check for PostgreSQL JSON operators if t.pos.Index < len(t.input) { nextR, nextSize := utf8.DecodeRune(t.input[t.pos.Index:])