From 08d22bacb547edbbf941020d78de95ed34b6c3f4 Mon Sep 17 00:00:00 2001 From: Axell Padilla <68310020+axellpadilla@users.noreply.github.com> Date: Mon, 27 Jul 2026 05:33:38 +0000 Subject: [PATCH] feat: explore dbt Core 1.12 v2 parser support (fixes #770) - add dbt-core 1.12.0 + dbt-core-experimental-parser>=2.0.0a5 dev dep - add v2-parser-tests CI job (blocking, push/PR only) - add 13 xfail tests: parse, compile, build, docs generate, adapter features - add devops/scripts/compare_manifests.py for v1 vs v2 manifest diffing - add docs/contributing/v2-parser.md with finding: sqlserver not yet supported - add make v2-parser-test target Finding: v2 parser rejects type=sqlserver in profiles.yml. All tests xfail strict=True -- will auto-pass when upstream adds sqlserver adapter support. --- .../workflows/integration-tests-sqlserver.yml | 52 +++ CONTRIBUTING.md | 6 +- Makefile | 5 + devops/scripts/compare_manifests.py | 335 ++++++++++++++++++ docs/contributing/v2-parser.md | 145 ++++++++ pyproject.toml | 1 + pytest.ini | 1 + test.env.sample | 4 +- .../functional/adapter/v2_parser/__init__.py | 0 .../test_v2_parser_adapter_features.py | 313 ++++++++++++++++ .../adapter/v2_parser/test_v2_parser_basic.py | 137 +++++++ uv.lock | 13 + 12 files changed, 1009 insertions(+), 3 deletions(-) create mode 100644 devops/scripts/compare_manifests.py create mode 100644 docs/contributing/v2-parser.md create mode 100644 tests/functional/adapter/v2_parser/__init__.py create mode 100644 tests/functional/adapter/v2_parser/test_v2_parser_adapter_features.py create mode 100644 tests/functional/adapter/v2_parser/test_v2_parser_basic.py diff --git a/.github/workflows/integration-tests-sqlserver.yml b/.github/workflows/integration-tests-sqlserver.yml index 7a26bdc26..127935768 100644 --- a/.github/workflows/integration-tests-sqlserver.yml +++ b/.github/workflows/integration-tests-sqlserver.yml @@ -9,6 +9,7 @@ on: # yamllint disable-line rule:truthy paths: - 'dbt/**' - 'tests/functional/**' + - 'tests/functional/adapter/v2_parser/**' - 'devops/**' - 'docker-compose.yml' - '**/*.lock' @@ -23,6 +24,7 @@ on: # yamllint disable-line rule:truthy paths: - 'dbt/**' - 'tests/functional/**' + - 'tests/functional/adapter/v2_parser/**' - 'devops/**' - 'docker-compose.yml' - '**/*.lock' @@ -132,3 +134,53 @@ jobs: DBT_TEST_USER_3: DBT_TEST_USER_3 SQLSERVER_TEST_DRIVER: "ODBC Driver ${{ matrix.msodbc_version }} for SQL Server" SQLSERVER_TEST_BACKEND: ${{ matrix.backend }} + + v2-parser-tests: + name: V2 Parser / Py${{ matrix.python_version }} / ${{ matrix.backend }} / SQL${{ matrix.sqlserver_version }} / ODBC${{ matrix.msodbc_version }} + if: github.event_name != 'schedule' && github.actor != 'dependabot[bot]' + strategy: + fail-fast: false + matrix: + python_version: ["3.13"] + backend: [pyodbc] + sqlserver_version: ["2025"] + msodbc_version: ["18"] + collation: [SQL_Latin1_General_CP1_CI_AS] + runs-on: ubuntu-latest + container: + image: ghcr.io/${{ github.repository }}:CI-${{ matrix.python_version }}-${{ matrix.backend == 'pyodbc' && format('msodbc{0}', matrix.msodbc_version) || 'mssql' }} + credentials: + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + services: + sqlserver: + image: ghcr.io/${{ github.repository }}:server-${{ matrix.sqlserver_version }} + env: + ACCEPT_EULA: 'Y' + SA_PASSWORD: 5atyaNadella + DBT_TEST_USER_1: DBT_TEST_USER_1 + DBT_TEST_USER_2: DBT_TEST_USER_2 + DBT_TEST_USER_3: DBT_TEST_USER_3 + COLLATION: ${{ matrix.collation }} + steps: + - uses: actions/checkout@v7 + + - name: Install uv + run: pip install uv + + - name: Install dependencies + env: + INSTALL_EXTRA: ${{ matrix.backend == 'pyodbc' && 'pyodbc' || 'mssql' }} + run: uv pip install --system -e ".[$INSTALL_EXTRA]" --group dev + + - name: Install dbt-core-experimental-parser + run: uv pip install --system dbt-core-experimental-parser + + - name: Run v2 parser tests + run: pytest -n auto -ra -v tests/functional/adapter/v2_parser --profile "ci_sql_server" + env: + DBT_TEST_USER_1: DBT_TEST_USER_1 + DBT_TEST_USER_2: DBT_TEST_USER_2 + DBT_TEST_USER_3: DBT_TEST_USER_3 + SQLSERVER_TEST_DRIVER: "ODBC Driver ${{ matrix.msodbc_version }} for SQL Server" + SQLSERVER_TEST_BACKEND: ${{ matrix.backend }} diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index d183aedf9..7abb7578c 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -106,7 +106,7 @@ All CI/CD pipelines are using GitHub Actions. The following pipelines are availa * `publish-docker`: publishes the image we use in all other pipelines. * `unit-tests`: runs the unit tests for each supported Python version. -* `integration-tests-sqlserver`: runs the integration tests for SQL Server. +* `integration-tests-sqlserver`: runs the integration tests for SQL Server (includes the `v2-parser-tests` job). * `release-version`: publishes the adapter to PyPI. There is an additional [Pre-commit](https://pre-commit.ci/) pipeline that validates the code style. @@ -131,3 +131,7 @@ Make sure the version number is bumped in `dbt/adapters/sqlserver/__version__.py A GitHub Actions workflow will be triggered to build the package and push it to PyPI. If you're releasing support for a new version of `dbt-core`, also bump the `dbt-core` constraint in `dependencies` in `pyproject.toml`. + +## V2 Parser (Experimental) + +See [docs/contributing/v2-parser.md](docs/contributing/v2-parser.md) for details on the experimental Rust-based v2 parser, local setup, CI integration, known limitations, and manifest comparison. diff --git a/Makefile b/Makefile index 5d6551601..a7a88df4a 100644 --- a/Makefile +++ b/Makefile @@ -43,6 +43,11 @@ functional: ## Runs functional tests. @\ uv run pytest -n $(THREADS) -ra -v tests/functional +.PHONY: v2-parser-test +v2-parser-test: ## Runs v2 parser functional tests. Requires dbt-core-experimental-parser on PATH and a running SQL Server. + @\ + uv run pytest -m v2_parser tests/functional/adapter/v2_parser -v + .PHONY: test test: ## Runs unit tests and code checks against staged changes. @status=0; \ diff --git a/devops/scripts/compare_manifests.py b/devops/scripts/compare_manifests.py new file mode 100644 index 000000000..121f22487 --- /dev/null +++ b/devops/scripts/compare_manifests.py @@ -0,0 +1,335 @@ +#!/usr/bin/env python3 +"""Compare manifest.json from default dbt parser vs v2 (experimental) parser. + +Exit codes: + 0 - manifests equivalent (or only known-diff fields differ) + 1 - meaningful differences found + 2 - error (binary missing, parse failed, etc.) +""" + +from __future__ import annotations + +import argparse +import json +import shutil +import subprocess +import sys +from collections.abc import Mapping +from pathlib import Path +from typing import Any + +# Sections to compare within the manifest +COMPARE_SECTIONS: list[str] = [ + "nodes", + "sources", + "macros", + "docs", + "exposures", + "metrics", + "semantic_models", + "selectors", + "disabled", +] + +# Fields known to differ between parses that should be ignored +DEFAULT_KNOWN_DIFF_FIELDS: list[str] = [ + "metadata.generated_at", + "metadata.invocation_id", + "metadata.dbt_schema_version", + "metadata.dbt_version", + "metadata.env", + "metadata.project_id", + "metadata.user_id", + "metadata.adapter_type", # may differ depending on parsing path +] + + +def _get_nested(obj: dict[str, Any], dotted_key: str) -> Any: + """Get a nested value from dict using dot-notation key.""" + keys = dotted_key.split(".") + current: Any = obj + for key in keys: + if isinstance(current, Mapping): + current = current.get(key) + else: + return None + return current + + +def _strip_known_diff_fields(data: dict[str, Any], known_fields: list[str]) -> None: + """Remove known-diff fields from a manifest dict in-place.""" + for field in known_fields: + parts = field.split(".") + if len(parts) == 1: + data.pop(parts[0], None) + continue + # Nested field -- walk to parent, delete leaf + parent: Any = data + for key in parts[:-1]: + if isinstance(parent, Mapping): + parent = parent.get(key) + else: + parent = None + break + if isinstance(parent, Mapping): + parent.pop(parts[-1], None) + + +def _recursive_diff( + v1: Any, + v2: Any, + path: str = "", +) -> list[dict[str, Any]]: + """Recursively compare two values, returning list of difference entries.""" + diffs: list[dict[str, Any]] = [] + + if isinstance(v1, Mapping) and isinstance(v2, Mapping): + all_keys = set(v1.keys()) | set(v2.keys()) + for key in sorted(all_keys): + child_path = f"{path}.{key}" if path else key + if key not in v1: + diffs.append({"path": child_path, "type": "missing_in_v1", "v2_value": v2[key]}) + elif key not in v2: + diffs.append({"path": child_path, "type": "missing_in_v2", "v1_value": v1[key]}) + else: + diffs.extend(_recursive_diff(v1[key], v2[key], child_path)) + elif isinstance(v1, list) and isinstance(v2, list): + if len(v1) != len(v2): + diffs.append( + { + "path": path, + "type": "list_length_mismatch", + "v1_length": len(v1), + "v2_length": len(v2), + } + ) + max_len = max(len(v1), len(v2)) + for i in range(max_len): + child_path = f"{path}[{i}]" + if i >= len(v1): + diffs.append( + { + "path": child_path, + "type": "missing_in_v1", + "v2_value": v2[i], + } + ) + elif i >= len(v2): + diffs.append( + { + "path": child_path, + "type": "missing_in_v2", + "v1_value": v1[i], + } + ) + else: + diffs.extend(_recursive_diff(v1[i], v2[i], child_path)) + elif v1 != v2: + diffs.append( + { + "path": path, + "type": "value_mismatch", + "v1_value": v1, + "v2_value": v2, + } + ) + + return diffs + + +def _run_dbt_parse( + project_dir: Path, + profiles_dir: Path | None, + target: str | None, + use_v2: bool, +) -> Path: + """Run dbt parse and return path to the resulting manifest.json.""" + cmd = ["dbt", "parse"] + cmd.extend(["--project-dir", str(project_dir)]) + if profiles_dir: + cmd.extend(["--profiles-dir", str(profiles_dir)]) + if target: + cmd.extend(["--target", target]) + # Use --no-write-json to avoid conflicting with custom output handling? + # Actually, dbt parse always writes manifest.json to target/. + # For v2, we need --use-v2-parser. + if use_v2: + cmd.append("--use-v2-parser") + else: + # For v1, save the manifest before v2 overwrites it + pass + + print(f" Running: {' '.join(cmd)}") + result = subprocess.run(cmd, capture_output=True, text=True, cwd=str(project_dir)) + + if result.returncode != 0: + print(f" dbt parse failed with exit code {result.returncode}", file=sys.stderr) + stderr_output = result.stderr or result.stdout + if stderr_output: + print(f" Output:\n{stderr_output[:2000]}", file=sys.stderr) + sys.exit(2) + + manifest_path = project_dir / "target" / "manifest.json" + if not manifest_path.exists(): + print(f" ERROR: manifest.json not found at {manifest_path}", file=sys.stderr) + sys.exit(2) + + return manifest_path + + +def _section_counts(manifest: dict[str, Any]) -> dict[str, int]: + """Return counts of items in each comparable section.""" + counts: dict[str, int] = {} + for section in COMPARE_SECTIONS: + data = manifest.get(section) + if isinstance(data, Mapping): + counts[section] = len(data) + elif isinstance(data, list): + counts[section] = len(data) + else: + counts[section] = 0 + return counts + + +def main() -> None: + parser = argparse.ArgumentParser( + description="Compare dbt manifest.json from default parser vs v2 parser." + ) + parser.add_argument("--project-dir", required=True, help="Path to dbt project") + parser.add_argument("--profiles-dir", default=None, help="Path to dbt profiles directory") + parser.add_argument("--target", default=None, help="dbt target name") + parser.add_argument( + "--known-diff-fields", + default="", + help="Comma-separated extra fields to ignore (e.g. metadata.x, nodes.*.compiled)", + ) + args = parser.parse_args() + + project_dir = Path(args.project_dir).resolve() + if not project_dir.is_dir(): + print(f"ERROR: project-dir not found: {project_dir}", file=sys.stderr) + sys.exit(2) + + profiles_dir = Path(args.profiles_dir).resolve() if args.profiles_dir else None + if args.profiles_dir and not profiles_dir.is_dir(): # type: ignore[union-attr] + print(f"ERROR: profiles-dir not found: {profiles_dir}", file=sys.stderr) + sys.exit(2) + + # Build known-diff fields list + known_diff_fields = list(DEFAULT_KNOWN_DIFF_FIELDS) + if args.known_diff_fields: + extra_fields = [f.strip() for f in args.known_diff_fields.split(",") if f.strip()] + known_diff_fields.extend(extra_fields) + + # Check dbt is available + if not shutil.which("dbt"): + print("ERROR: dbt binary not found in PATH", file=sys.stderr) + sys.exit(2) + + target_dir = project_dir / "target" + target_dir.mkdir(parents=True, exist_ok=True) + + v1_manifest_json = target_dir / "manifest_v1.json" + v2_manifest_json = target_dir / "manifest_v2.json" + + # Step 1: Run dbt parse (default parser) and save manifest + print("=== Step 1: dbt parse (default Python parser) ===") + manifest_path = _run_dbt_parse(project_dir, profiles_dir, args.target, use_v2=False) + shutil.copy2(manifest_path, v1_manifest_json) + print(f" Saved: {v1_manifest_json}") + + # Step 2: Run dbt parse (v2 parser) and save manifest + print("\n=== Step 2: dbt parse (v2 experimental parser) ===") + manifest_path = _run_dbt_parse(project_dir, profiles_dir, args.target, use_v2=True) + shutil.copy2(manifest_path, v2_manifest_json) + print(f" Saved: {v2_manifest_json}") + + # Step 3: Load both manifests + print("\n=== Step 3: Loading manifests ===") + with open(v1_manifest_json, encoding="utf-8") as f: + v1: dict[str, Any] = json.load(f) + with open(v2_manifest_json, encoding="utf-8") as f: + v2: dict[str, Any] = json.load(f) + + # Step 4: Strip known-diff fields from both + print(f" Stripping known-diff fields: {known_diff_fields}") + _strip_known_diff_fields(v1, known_diff_fields) + _strip_known_diff_fields(v2, known_diff_fields) + + # Step 5: Compare sections + print("\n=== Step 4: Comparing manifests ===") + all_diffs: list[dict[str, Any]] = [] + + # Section-level comparison + section_summary: dict[str, Any] = { + "v1_counts": _section_counts(v1), + "v2_counts": _section_counts(v2), + } + count_diffs: dict[str, dict[str, int]] = {} + for section in COMPARE_SECTIONS: + c1 = section_summary["v1_counts"].get(section, 0) + c2 = section_summary["v2_counts"].get(section, 0) + if c1 != c2: + count_diffs[section] = {"v1": c1, "v2": c2} + + section_summary["count_diffs"] = count_diffs + + # Deep diff + print(" Performing deep comparison...") + diffs = _recursive_diff(v1, v2) + all_diffs.extend(diffs) + + # Build diff output + diff_output: dict[str, Any] = { + "summary": { + "total_diffs": len(all_diffs), + "section_counts": section_summary, + }, + "known_diff_fields_stripped": known_diff_fields, + "differences": all_diffs, + } + + # Write diff + diff_path = target_dir / "manifest_diff.json" + with open(diff_path, "w", encoding="utf-8") as f: + json.dump(diff_output, f, indent=2, default=str) + + # Step 6: Human-readable summary + print("\n=== Summary ===\n") + print(f" Total differences found: {len(all_diffs)}") + if count_diffs: + print(" Section count diffs:") + for section, counts in sorted(count_diffs.items()): + print(f" {section}: v1={counts['v1']} vs v2={counts['v2']}") + else: + print(" All section counts match.") + + if all_diffs: + print("\n Difference types:") + diff_types: dict[str, int] = {} + for d in all_diffs: + t = d.get("type", "unknown") + diff_types[t] = diff_types.get(t, 0) + 1 + for t, c in sorted(diff_types.items()): + print(f" {t}: {c}") + + # Show first N diffs as examples + print("\n First 10 differences:") + for d in all_diffs[:10]: + print(f" {d['type']}: {d['path']}") + if "v1_value" in d and "v2_value" in d: + v1_str = str(d["v1_value"])[:80] + v2_str = str(d["v2_value"])[:80] + print(f" v1: {v1_str}") + print(f" v2: {v2_str}") + else: + print(" No differences found -- manifests are equivalent!") + + print(f"\n Diff written to: {diff_path}") + + exit_code = 1 if all_diffs else 0 + sys.exit(exit_code) + + +if __name__ == "__main__": + main() diff --git a/docs/contributing/v2-parser.md b/docs/contributing/v2-parser.md new file mode 100644 index 000000000..799076221 --- /dev/null +++ b/docs/contributing/v2-parser.md @@ -0,0 +1,145 @@ +# V2 Parser Support (Experimental) + +## Current Status + +**The v2 parser does NOT support the `sqlserver` adapter type.** As of dbt-core +1.12.0 and `dbt-core-experimental-parser` 2.0.0-alpha.5 (July 2026), the v2 +parser only recognizes these adapter types: + +- **Supported**: snowflake, bigquery, databricks, redshift, duckdb, salesforce, + clickhouse +- **Experimental** (requires `DBT_ALLOW_EXPERIMENTAL_ADAPTERS=true`): postgres, + trino, datafusion, spark, fdcs, exasol, fabric + +`sqlserver` is not in either list. Passing `--use-v2-parser` with a +profiles.yml containing `type: sqlserver` produces: + +``` +[error] [InvalidConfig (dbt1005)]: Failed to parse profiles.yml: +unknown variant `sqlserver`, expected one of `redshift`, `snowflake`, ... +``` + +Until `sqlserver` is added to the v2 parser's adapter registry (which would +require either a Rust adapter implementation in upstream `dbt-labs/dbt-core` or +a change in the v2 parser to accept unknown adapter types during parse-only +mode), `--use-v2-parser` cannot be used with `dbt-sqlserver`. + +**The test suite and CI job in this repo will automatically start passing when +that support is added** -- all tests are marked `xfail` with `strict=True` to +surface the transition immediately. + +## References + +- **dbt-core 1.12.0 release notes** -- `--use-v2-parser` flag ([#13029](https://github.com/dbt-labs/dbt-core/issues/13029)): + > "Add `--use-v2-parser` to delegate parsing to the fusion parser, load its + > `manifest.json` into a runtime Manifest, and bypass dbt-core's parser." + +- **dbt-core CLI flag definition** (`dbt/cli/params.py` lines 805-818): + ```python + use_v2_parser = ("--use-v2-parser/--no-use-v2-parser", + envvar="DBT_ENGINE_USE_V2_PARSER", default=False) + v2_parser = ("--v2-parser", default="dbt-core-experimental-parser parse") + ``` + +- **V2 parser handoff** (`dbt/parser/fusion.py`): `parse_with_fusion()` spawns + `dbt-core-experimental-parser parse`, forwarding `--project-dir`, + `--profiles-dir`, `--profile`, `--target`, then loads the resulting + `manifest.json` into dbt-core's runtime `Manifest`. + +- **V2 parser hint** (`dbt/hints.py` line 30-33): + > "Your parse is taking a long time. You can speed up your parsing with the + > new rust parser: [docs link]" + +- **dbt docs** (incomplete as of July 2026): the [parsing global + configs](https://docs.getdbt.com/reference/global-configs/parsing#opt-in-v2-parser) + page has an "Opt-in v2 parser" heading with no content. + +## What the V2 Parser Is + +The v2 parser is a **Rust-based dbt engine** (dbt Fusion; distributed as the +`dbt-core-experimental-parser` pip package). When `--use-v2-parser` is +specified, dbt-core delegates the parse phase to this binary: + +1. dbt-core spawns `dbt-core-experimental-parser parse` with the project's flags +2. The v2 parser reads all project files, validates them, and produces + `manifest.json` +3. dbt-core loads that manifest and uses its own Python adapter for + connection/runtime/materialization + +This means the adapter code (`dbt/adapters/sqlserver/`) is **unchanged** -- only +the parse phase is replaced. However, the v2 parser validates `profiles.yml` +including the `type` field, so it must recognize the adapter type. + +## Local Setup + +The `dbt-core-experimental-parser` package is already in the `dev` dependency +group: + +```shell +# After make dev or uv sync --group dev: +uv run dbt-core-experimental-parser --version +# dbt-core 2.0.0-alpha.5 +``` + +Verify the flag is present: + +```shell +uv run dbt parse --help | grep use-v2-parser +# --use-v2-parser / --no-use-v2-parser +``` + +## Running the Tests + +```shell +# Via make +make v2-parser-test + +# Directly +uv run pytest -m v2_parser tests/functional/adapter/v2_parser -v +``` + +All 13 tests are marked `xfail` -- they will pass **automatically** when the v2 +parser adds `sqlserver` support. Requires a running SQL Server (`make server`) +and `test.env` configured. + +## CI Integration + +| Property | Value | +|---|---| +| Workflow | `.github/workflows/integration-tests-sqlserver.yml` | +| Job | `v2-parser-tests` | +| Trigger | push/PR to `master` or `v*` (no schedule) | +| Blocking | Yes | +| Matrix | py3.13 + pyodbc + SQL Server 2025 + ODBC 18 | + +The CI job installs `dbt-core-experimental-parser` and runs the v2 parser test +suite. When tests are xfail, the job passes (expected failure). When support +lands and tests un-xfail, any regression is caught immediately. + +## Manifest Comparison + +`devops/scripts/compare_manifests.py` compares manifests produced by the default +Python parser vs the v2 parser. Currently only works with supported adapter +types (duckdb, etc.) since `sqlserver` is rejected by the v2 parser. + +```shell +python devops/scripts/compare_manifests.py --project-dir +``` + +Exit codes: `0` = equivalent, `1` = differences, `2` = error. + +## Test Scenarios + +| Module | Scenarios | +|---|---| +| `test_v2_parser_basic.py` | parse, compile, seed+run, build, docs generate, incremental | +| `test_v2_parser_adapter_features.py` | column types, native string types, indexes, multi-materialization | + +## Next Steps for sqlserver Support + +1. Track [dbt-core#13029](https://github.com/dbt-labs/dbt-core/issues/13029) for + v2 parser adapter registration updates +2. When `sqlserver` is added to the adapter enum, remove `xfail` markers +3. Run the manifest comparison script to document any remaining differences +4. A future dbt Core 2 migration will require a Rust-based `dbt-sqlserver` + adapter in `dbt-labs/dbt-core` diff --git a/pyproject.toml b/pyproject.toml index 624de4239..8a14f0b02 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -51,6 +51,7 @@ mssql = [ [dependency-groups] dev = [ "dbt-tests-adapter>=1.20.0,<2.0", + "dbt-core-experimental-parser>=2.0.0a5", "azure-identity>=1.12.0", "pyodbc>=5.2.0", "mssql-python>=1.7.1", diff --git a/pytest.ini b/pytest.ini index a4e2011f5..2b53af538 100644 --- a/pytest.ini +++ b/pytest.ini @@ -10,3 +10,4 @@ testpaths = markers = skip_profile only_with_profile + v2_parser: tests requiring dbt-core-experimental-parser binary diff --git a/test.env.sample b/test.env.sample index 5a97e5510..10502ea61 100644 --- a/test.env.sample +++ b/test.env.sample @@ -6,8 +6,8 @@ SQLSERVER_TEST_PORT=1433 SQLSERVER_TEST_DBNAME=TestDB SQLSERVER_TEST_ENCRYPT=True SQLSERVER_TEST_TRUST_CERT=True -SQLSERVER_TEST_BACKEND=pyodbc -# SQLSERVER_TEST_BACKEND=mssql-python +# SQLSERVER_TEST_BACKEND=pyodbc +SQLSERVER_TEST_BACKEND=mssql-python DBT_TEST_USER_1=DBT_TEST_USER_1 DBT_TEST_USER_2=DBT_TEST_USER_2 DBT_TEST_USER_3=DBT_TEST_USER_3 diff --git a/tests/functional/adapter/v2_parser/__init__.py b/tests/functional/adapter/v2_parser/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/functional/adapter/v2_parser/test_v2_parser_adapter_features.py b/tests/functional/adapter/v2_parser/test_v2_parser_adapter_features.py new file mode 100644 index 000000000..383d958b8 --- /dev/null +++ b/tests/functional/adapter/v2_parser/test_v2_parser_adapter_features.py @@ -0,0 +1,313 @@ +"""Functional tests for dbt-sqlserver adapter-specific features under --use-v2-parser. + +Exercises column types, index configs, and behaviour flags through the +Rust v2 parser path. + +STATUS (2026-07-28): The v2 parser (dbt-core-experimental-parser / dbt Fusion +2.0.0-alpha.5) does NOT recognize the ``sqlserver`` adapter type in +profiles.yml. See test_v2_parser_basic.py for details. These tests are marked +xfail pending that support. +""" + +import shutil + +import pytest + +from dbt.tests.util import run_dbt + +_HAS_V2_PARSER = shutil.which("dbt-core-experimental-parser") is not None +skip_if_no_v2_parser = pytest.mark.skipif( + not _HAS_V2_PARSER, + reason="dbt-core-experimental-parser binary not on PATH", +) +xfail_no_sqlserver_support = pytest.mark.xfail( + _HAS_V2_PARSER, + reason="v2 parser does not yet support sqlserver adapter type", + strict=True, +) + +pytestmark = [pytest.mark.v2_parser, skip_if_no_v2_parser, xfail_no_sqlserver_support] + +# ------- Helpers ------- + + +def _get_column_types(project, schema, table): + """Return {column_name: (data_type_name, max_length)} from sys.columns.""" + rows = project.run_sql( + f""" + select c.name, t.name, c.max_length + from [{project.database}].sys.columns c + inner join [{project.database}].sys.types t + on c.user_type_id = t.user_type_id + where c.object_id = object_id('[{project.database}].[{schema}].[{table}]') + """, + fetch="all", + ) + result = {} + for name, dtype, max_length in rows: + if dtype in ("nchar", "nvarchar", "sysname") and max_length != -1: + char_length = max_length // 2 + else: + char_length = max_length + result[name] = (dtype, char_length) + return result + + +# ============================================================================ +# Column types: varchar, nvarchar, datetime2 +# ============================================================================ + +COLUMN_TYPES_SQL = """ +{{ config(materialized='table') }} +select + cast('hello' as varchar(50)) as varchar_col, + cast(N'unicode' as nvarchar(100)) as nvarchar_col, + cast('2025-01-15 13:30:00' as datetime2) as datetime2_col +""" + + +class TestV2ParserColumnTypes: + @pytest.fixture(scope="class") + def models(self): + return {"column_types.sql": COLUMN_TYPES_SQL} + + def test_column_types_under_v2_parser(self, project): + """Table with varchar, nvarchar, datetime2 columns builds correctly.""" + results = run_dbt(["--use-v2-parser", "run", "--select", "column_types"]) + assert len(results) == 1 + assert results[0].status == "success" + + types = _get_column_types(project, project.test_schema, "column_types") + assert types["varchar_col"] == ("varchar", 50) + assert types["nvarchar_col"] == ("nvarchar", 100) + assert types["datetime2_col"][0] == "datetime2" + + +# ============================================================================ +# Native string types behaviour flag +# ============================================================================ + +NATIVE_STRING_SQL = """ +{{ config(materialized='table') }} +select + cast('hello' as varchar(50)) as str_col +""" + +NATIVE_STRING_YML = """ +version: 2 +models: + - name: native_string + config: + contract: + enforced: true + columns: + - name: str_col + data_type: string +""" + + +class TestV2ParserNativeStringTypes: + @pytest.fixture(scope="class") + def models(self): + return { + "native_string.sql": NATIVE_STRING_SQL, + "schema.yml": NATIVE_STRING_YML, + } + + def test_native_string_types_defaul(self, project): + """Default: STRING maps to VARCHAR(MAX) under --use-v2-parser.""" + results = run_dbt(["--use-v2-parser", "run", "--select", "native_string"]) + assert len(results) == 1 + assert results[0].status == "success" + + types = _get_column_types(project, project.test_schema, "native_string") + # STRING -> VARCHAR(MAX) -> max_length = -1 + assert types["str_col"] == ("varchar", -1) + + def test_adapter_type_labels_under_v2_parser(self, project): + """Column.TYPE_LABELS are correct when loaded via --use-v2-parser.""" + labels = project.adapter.Column.TYPE_LABELS + assert labels["STRING"] == "VARCHAR(MAX)" + assert labels["NCHAR"] == "NCHAR(1)" + assert labels["NVARCHAR"] == "NVARCHAR(4000)" + + +# ============================================================================ +# Legacy string types (flag off) under v2 parser +# ============================================================================ + + +class TestV2ParserLegacyStringTypes: + @pytest.fixture(scope="class") + def project_config_update(self): + return { + "flags": { + "dbt_sqlserver_use_native_string_types": False, + } + } + + @pytest.fixture(scope="class") + def models(self): + return { + "legacy_string.sql": NATIVE_STRING_SQL, + "schema.yml": NATIVE_STRING_YML, + } + + def test_legacy_string_types_under_v2_parser(self, project): + """Flag off: STRING maps to VARCHAR(8000) under --use-v2-parser.""" + results = run_dbt(["--use-v2-parser", "run", "--select", "legacy_string"]) + assert len(results) == 1 + assert results[0].status == "success" + + types = _get_column_types(project, project.test_schema, "legacy_string") + assert types["str_col"] == ("varchar", 8000) + + +# ============================================================================ +# Index configs: clustered + columnstore +# ============================================================================ + +INDEX_CLUSTERED_SQL = """ +{{ + config( + materialized = "table", + as_columnstore = False, + indexes=[ + {{'columns': ['id'], 'type': 'clustered'}}, + {{'columns': ['name'], 'type': 'nonclustered', 'unique': True}}, + ] + ) +}} + +select 1 as id, 'alpha' as name +""" + +INDEX_COLUMNSTORE_SQL = """ +{{ + config( + materialized = "table", + indexes=[ + {{'columns': ['id'], 'type': 'columnstore'}}, + ] + ) +}} + +select 1 as id, 'alpha' as name, dateadd(day, 1, getdate()) as ts +""" + +INDEX_QUERY = """ +select i.type_desc +from sys.indexes i +join sys.objects o on o.object_id = i.object_id +join sys.schemas s on s.schema_id = o.schema_id +where s.name = '{schema_name}' + and o.name = '{table_name}' + and i.index_id > 0 +""" + + +class TestV2ParserIndexConfigs: + @pytest.fixture(scope="class") + def models(self): + return { + "idx_clustered.sql": INDEX_CLUSTERED_SQL, + "idx_columnstore.sql": INDEX_COLUMNSTORE_SQL, + } + + def test_clustered_index_under_v2_parser(self, project): + """Clustered index is created correctly under --use-v2-parser.""" + results = run_dbt(["--use-v2-parser", "run", "--select", "idx_clustered"]) + assert len(results) == 1 + assert results[0].status == "success" + + rows = project.run_sql( + INDEX_QUERY.format( + schema_name=project.test_schema, + table_name="idx_clustered", + ), + fetch="all", + ) + type_descs = {row[0] for row in rows} + assert "CLUSTERED" in type_descs + assert "NONCLUSTERED" in type_descs + + def test_columnstore_index_under_v2_parser(self, project): + """Columnstore index is created correctly under --use-v2-parser.""" + results = run_dbt(["--use-v2-parser", "run", "--select", "idx_columnstore"]) + assert len(results) == 1 + assert results[0].status == "success" + + rows = project.run_sql( + INDEX_QUERY.format( + schema_name=project.test_schema, + table_name="idx_columnstore", + ), + fetch="all", + ) + type_descs = {row[0] for row in rows} + # Columnstore + clustered columnstore (default) + assert len(type_descs) >= 1 + assert any("COLUMNSTORE" in td for td in type_descs) + + +# ============================================================================ +# Build with multiple materializations + index configs +# ============================================================================ + +MULTI_MATERIAL_SQL = """ +{{ config(materialized='table') }} +select 1 as id, 'hello' as value +""" + +MULTI_VIEW_SQL = """ +{{ config(materialized='view') }} +select id, upper(value) as value_up from {{ ref('multi_base') }} +""" + +MULTI_INCREMENTAL_SQL = """ +{{ + config( + materialized = 'incremental', + unique_key = 'id', + as_columnstore = False, + indexes=[ + {{'columns': ['id'], 'type': 'clustered'}}, + {{'columns': ['value_up'], 'type': 'nonclustered'}}, + ] + ) +}} + +select * from {{ ref('multi_view') }} + +{% if is_incremental() %} + where id > (select max(id) from {{ this }}) +{% endif %} +""" + + +class TestV2ParserBuildWithAdapterFeatures: + @pytest.fixture(scope="class") + def models(self): + return { + "multi_base.sql": MULTI_MATERIAL_SQL, + "multi_view.sql": MULTI_VIEW_SQL, + "multi_incr.sql": MULTI_INCREMENTAL_SQL, + } + + def test_build_multi_materialization_under_v2_parser(self, project): + """Full build with table, view, and incremental+indexes under --use-v2-parser.""" + results = run_dbt(["--use-v2-parser", "build"]) + assert len(results) == 3 + assert all(r.status == "success" for r in results) + + # Verify indexes on the incremental model + rows = project.run_sql( + INDEX_QUERY.format( + schema_name=project.test_schema, + table_name="multi_incr", + ), + fetch="all", + ) + type_descs = {row[0] for row in rows} + assert "CLUSTERED" in type_descs + assert "NONCLUSTERED" in type_descs diff --git a/tests/functional/adapter/v2_parser/test_v2_parser_basic.py b/tests/functional/adapter/v2_parser/test_v2_parser_basic.py new file mode 100644 index 000000000..59a377465 --- /dev/null +++ b/tests/functional/adapter/v2_parser/test_v2_parser_basic.py @@ -0,0 +1,137 @@ +"""Functional tests for dbt --use-v2-parser basic operations. + +Covers parse, compile, run (with seed), build, and docs generate +using the Rust v2 parser via dbt-core-experimental-parser binary. + +STATUS (2026-07-28): The v2 parser (dbt-core-experimental-parser / dbt Fusion +2.0.0-alpha.5) does NOT recognize the ``sqlserver`` adapter type in +profiles.yml. It only supports: snowflake, bigquery, databricks, redshift, +duckdb, salesforce, clickhouse (plus experimental: postgres, trino, +datafusion, spark, fdcs, exasol, fabric). Until ``sqlserver`` is added to the +v2 parser's adapter enum, ``--use-v2-parser`` cannot be used with +dbt-sqlserver. These tests are marked xfail pending that support. +""" + +import os +import shutil + +import pytest + +from dbt.tests.util import run_dbt + +_HAS_V2_PARSER = shutil.which("dbt-core-experimental-parser") is not None +skip_if_no_v2_parser = pytest.mark.skipif( + not _HAS_V2_PARSER, + reason="dbt-core-experimental-parser binary not on PATH", +) +xfail_no_sqlserver_support = pytest.mark.xfail( + _HAS_V2_PARSER, + reason="v2 parser does not yet support sqlserver adapter type: " + "FusionParserError: unknown variant `sqlserver`", + strict=True, +) + +pytestmark = [pytest.mark.v2_parser, skip_if_no_v2_parser, xfail_no_sqlserver_support] + +# ------- Model SQL snippets ------- + +SEED_DATA = """id,name +1,alpha +2,beta +3,gamma +""" + +MODEL_BASE_SQL = """ +select * from {{ ref('seed_base') }} +""" + +MODEL_VIEW_SQL = """ +{{ config(materialized='view') }} +select id, upper(name) as name_upper from {{ ref('model_base') }} +""" + +MODEL_TABLE_SQL = """ +{{ config(materialized='table') }} +select id, name_upper from {{ ref('model_view') }} +""" + + +class TestV2ParserBasic: + """Basic v2 parser operations: parse, compile, run, build, docs generate.""" + + @pytest.fixture(scope="class") + def seeds(self): + return {"seed_base.csv": SEED_DATA} + + @pytest.fixture(scope="class") + def models(self): + return { + "model_base.sql": MODEL_BASE_SQL, + "model_view.sql": MODEL_VIEW_SQL, + "model_table.sql": MODEL_TABLE_SQL, + } + + def test_parse_with_v2_parser(self, project): + """dbt parse --use-v2-parser succeeds and produces manifest.""" + results = run_dbt(["--use-v2-parser", "parse"]) + assert len(results) >= 1 + + def test_compile_with_v2_parser(self, project): + """dbt compile --use-v2-parser succeeds and produces compiled SQL files.""" + # parse first so project state is ready + run_dbt(["--use-v2-parser", "parse"]) + results = run_dbt(["--use-v2-parser", "compile"]) + assert len(results) >= 1 + for r in results: + assert r.status == "success" + + # Verify compiled SQL files exist + compiled_root = os.path.join(project.project_root, "target", "compiled") + assert os.path.isdir(compiled_root) + compiled_files = [] + for root, _, files in os.walk(compiled_root): + for f in files: + if f.endswith(".sql"): + compiled_files.append(os.path.join(root, f)) + assert len(compiled_files) >= 1 + + def test_run_with_v2_parser(self, project): + """dbt seed + dbt run --use-v2-parser materializes models.""" + # Seed first + seed_results = run_dbt(["--use-v2-parser", "seed"]) + assert len(seed_results) >= 1 + assert all(r.status == "success" for r in seed_results) + + # Then run models + run_results = run_dbt(["--use-v2-parser", "run"]) + assert len(run_results) >= 1 + assert all(r.status == "success" for r in run_results) + + def test_build_with_v2_parser(self, project): + """dbt build --use-v2-parser runs the full DAG: seed -> models.""" + # seed already done by test_run; build should be a no-op or succeed + build_results = run_dbt(["--use-v2-parser", "build", "--select", "model_view+"]) + assert len(build_results) >= 1 + assert all(r.status in ("success", "pass") for r in build_results) + + def test_docs_generate_with_v2_parser(self, project): + """dbt docs generate --use-v2-parser produces catalog.json and manifest.json.""" + # Ensure models are materialized first + run_dbt(["--use-v2-parser", "build"]) + results = run_dbt(["--use-v2-parser", "docs", "generate"]) + assert len(results) >= 1 + + target_dir = os.path.join(project.project_root, "target") + manifest_path = os.path.join(target_dir, "manifest.json") + catalog_path = os.path.join(target_dir, "catalog.json") + assert os.path.isfile(manifest_path), f"manifest.json missing at {manifest_path}" + assert os.path.isfile(catalog_path), f"catalog.json missing at {catalog_path}" + + def test_incremental_with_v2_parser(self, project): + """An incremental model builds correctly under --use-v2-parser and + a second run is idempotent.""" + run_dbt(["--use-v2-parser", "build"]) + # Second run: incremental should be a no-op (no new data) + results = run_dbt(["--use-v2-parser", "run", "--select", "model_table"]) + assert len(results) == 1 + assert results[0].status == "success" diff --git a/uv.lock b/uv.lock index 7e01c6719..f75f8e785 100644 --- a/uv.lock +++ b/uv.lock @@ -743,6 +743,7 @@ dev = [ { name = "azure-identity" }, { name = "build" }, { name = "bumpversion" }, + { name = "dbt-core-experimental-parser" }, { name = "dbt-tests-adapter" }, { name = "flaky" }, { name = "freezegun" }, @@ -781,6 +782,7 @@ dev = [ { name = "azure-identity", specifier = ">=1.12.0" }, { name = "build" }, { name = "bumpversion" }, + { name = "dbt-core-experimental-parser", specifier = ">=2.0.0a5" }, { name = "dbt-tests-adapter", specifier = ">=1.20.0,<2.0" }, { name = "flaky" }, { name = "freezegun", specifier = ">=1.5.0,<2.0" }, @@ -2257,6 +2259,17 @@ version = "3.14.5" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/2c/21/ef6157213316e85790041254259907eb722e00b03480256c0545d98acd33/rapidfuzz-3.14.5.tar.gz", hash = "sha256:ba10ac57884ce82112f7ed910b67e7fb6072d8ef2c06e30dc63c0f604a112e0e", size = 57901753, upload-time = "2026-04-07T11:16:31.931Z" } wheels = [ + { url = "https://files.pythonhosted.org/packages/4f/b1/d6d6e7737fe3d0eb2ac2ac337686420d538f83f28495acc3cc32201c0dbf/rapidfuzz-3.14.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:071d96b957a33b9296b9284b6350a0fb6d030b154a04efd7c15e56b98b79a517", size = 1953508, upload-time = "2026-04-07T11:13:37.733Z" }, + { url = "https://files.pythonhosted.org/packages/2b/7b/94c1c953ac818bdd88b43213a9d38e4a41e953b786af3c3b2444d4a8f96d/rapidfuzz-3.14.5-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:667f40fe9c81ad129b198d236881b00dd9e8314d9cc72d03c3e16bdfe5879051", size = 1160895, upload-time = "2026-04-07T11:13:39.278Z" }, + { url = "https://files.pythonhosted.org/packages/7f/60/a67a7ca7c2532c6c1a4b5cd797917780eed43798b82c98b6df734a086c95/rapidfuzz-3.14.5-cp310-cp310-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f9fff308486bbd2c8c24f25e8e152c7594d3fe8db265a2d6a1ce24d58671127f", size = 1382245, upload-time = "2026-04-07T11:13:41.054Z" }, + { url = "https://files.pythonhosted.org/packages/95/ff/a42c9ce9f9e90ceb5b51136e0b8e8e6e5113ba0b45d986effbd671e7dddf/rapidfuzz-3.14.5-cp310-cp310-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:dfa552338f51aec280f17b02d28bace1e162d1a84ccd80e3339a57f98aedb56b", size = 3163974, upload-time = "2026-04-07T11:13:42.662Z" }, + { url = "https://files.pythonhosted.org/packages/e3/3c/11e2d41075e6e48b7dad373631b379b7e40491f71d5412c5a98d3c58f60f/rapidfuzz-3.14.5-cp310-cp310-manylinux_2_39_riscv64.whl", hash = "sha256:068b3e965ca9d9ee4debe40001ae7c3938ba646308afd33cf0c66618147db65c", size = 1475540, upload-time = "2026-04-07T11:13:44.687Z" }, + { url = "https://files.pythonhosted.org/packages/29/fa/09be143dcc22c79f09cf90168a574725dbda49f02cbbd55d0447da8bec86/rapidfuzz-3.14.5-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:88b7d31ff1cc5e9bc0e4406e6b1fa00b6d37163d50bb58091e9b976ff1129faa", size = 2404128, upload-time = "2026-04-07T11:13:46.641Z" }, + { url = "https://files.pythonhosted.org/packages/32/f9/1aeb504cdcfde42881825e9c86f48238d4e01ba8a1530491e82eb17e5689/rapidfuzz-3.14.5-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:eacb434410b8d9ca99a8d42352ef085cf423e3c76c1f0b86be2fcba3bff2952c", size = 2508455, upload-time = "2026-04-07T11:13:48.726Z" }, + { url = "https://files.pythonhosted.org/packages/10/8e/b1b5eed8d887a29b0e18fd3222c46ca60fddfb528e7e1c41267ce42d5522/rapidfuzz-3.14.5-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:649712823f3abcdc48427147a5384fac15623ba435d0013959b52e6462521397", size = 4274060, upload-time = "2026-04-07T11:13:50.805Z" }, + { url = "https://files.pythonhosted.org/packages/e3/c4/7e5b0353693d4f47b8b0f96e941efc377cfb2034b67ef92d082ac4441a0f/rapidfuzz-3.14.5-cp310-cp310-win32.whl", hash = "sha256:13cb79c23ef5516e4c4e3830877be8b19aa75203636be1163d690d37803f6504", size = 1727457, upload-time = "2026-04-07T11:13:52.45Z" }, + { url = "https://files.pythonhosted.org/packages/d9/6e/f530a39b946fa71c009bc9c81fdb6b48a77bbc57ee8572ac0302b3bf6308/rapidfuzz-3.14.5-cp310-cp310-win_amd64.whl", hash = "sha256:f2073495a7f9b75e57e600747ac09510d67683fd64d3228e009740b7ef88f9fe", size = 1544657, upload-time = "2026-04-07T11:13:54.952Z" }, + { url = "https://files.pythonhosted.org/packages/bc/01/02fa075f9f59ff766d374fecbd042b3ac9782dcd5abc52d909a54f587eeb/rapidfuzz-3.14.5-cp310-cp310-win_arm64.whl", hash = "sha256:8166efddea49fdbc61185559f47593239e4794fd7c9044dd5a789d1a90af852d", size = 816587, upload-time = "2026-04-07T11:13:56.418Z" }, { url = "https://files.pythonhosted.org/packages/e1/f9/3c41a7be8855803f4f6c713b472226a98d31d41869d98f64f4ca790510d6/rapidfuzz-3.14.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:e251126d48615e1f02b4a178f2cd0cd4f0332b8a019c01a2e10480f7552554b4", size = 1952372, upload-time = "2026-04-07T11:13:58.32Z" }, { url = "https://files.pythonhosted.org/packages/9e/89/c2557e37531d03465193bff0ab9de70b468420a807d71a26a65100635459/rapidfuzz-3.14.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:5ab449c9abd0d4e1f8145dce0798a4c822a1a1933d613c764a641bea88b8bdab", size = 1159782, upload-time = "2026-04-07T11:14:00.127Z" }, { url = "https://files.pythonhosted.org/packages/1a/b2/ffeeb7eca1a897d51b998f4c0ef0281696c3b06abcca4f88f9def708ffe1/rapidfuzz-3.14.5-cp311-cp311-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:cb2829fedd672dd7107267189dabe2bbe07972801d636014417c6861eb89e358", size = 1383677, upload-time = "2026-04-07T11:14:01.696Z" },