From 04adb042ee1dbc4a433c6cb0ba1710f96661493a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A5rten=20Pettersson?= Date: Wed, 15 Jul 2026 09:26:48 +0200 Subject: [PATCH 1/5] fix(indexes): ensure CCI created on final relation, not __dbt_tmp (#578) Add helper macros for index creation: mssql__quote_ident, mssql__qualified_relation, mssql__strip_dbt_suffix, mssql__index_name, and sqlserver__index_exists. The key fix: index names are now derived from the *final* relation name (with __dbt_tmp / __dbt_backup suffixes stripped). This ensures: * A second run produces an identical index name (idempotent) * CCIs are not orphaned after intermediate relations are renamed * No conflicts when `drop_all_indexes_on_table` runs Update create_clustered_index and create_nonclustered_index to accept a `relation` parameter and use bracket-quoted, ]]-escaped column names. Add comprehensive integration and unit tests covering incremental runs,INCLUDE columns, quoted identifiers, and idempotence. --- .../sqlserver/macros/adapters/indexes.sql | 126 +++++-- .../macros/relations/table/create.sql | 8 +- .../adapter/mssql/test_index_macros.py | 247 ++++++++++++ tests/unit/adapters/mssql/test_indexes.py | 355 ++++++++++++++++++ 4 files changed, 698 insertions(+), 38 deletions(-) create mode 100644 tests/unit/adapters/mssql/test_indexes.py diff --git a/dbt/include/sqlserver/macros/adapters/indexes.sql b/dbt/include/sqlserver/macros/adapters/indexes.sql index 7ce2be3db..2113900d3 100644 --- a/dbt/include/sqlserver/macros/adapters/indexes.sql +++ b/dbt/include/sqlserver/macros/adapters/indexes.sql @@ -1,17 +1,67 @@ +{% macro mssql__quote_ident(name) -%} + {{ return('[' ~ name | replace(']', ']]') ~ ']') }} +{%- endmacro %} + + +{% macro mssql__qualified_relation(rel) -%} + {{ return( + mssql__quote_ident(rel.database) ~ '.' ~ + mssql__quote_ident(rel.schema) ~ '.' ~ + mssql__quote_ident(rel.identifier) + ) }} +{%- endmacro %} + + +{% macro mssql__strip_dbt_suffix(identifier) -%} + {%- set ns = namespace(result=identifier) -%} + {%- for suffix in ['__dbt_tmp_vw', '__dbt_backup', '__dbt_tmp'] -%} + {%- if ns.result.endswith(suffix) -%} + {%- set ns.result = ns.result[:(ns.result | length) - (suffix | length)] -%} + {%- endif -%} + {%- endfor -%} + {{ return(ns.result) }} +{%- endmacro %} + + +{% macro mssql__index_name(rel, type, columns, unique=False, includes=false) -%} + {%- set cols = [columns] if columns is string else columns -%} + {%- set incs = [] if not includes else ([includes] if includes is string else includes) -%} + {%- set stripped = mssql__strip_dbt_suffix(rel.identifier) | replace('[', '') | replace(']', '') -%} + {%- set sig = type ~ '|' ~ (cols | join(',')) ~ '|' ~ (unique | string) ~ '|' ~ (incs | join(',')) -%} + {%- set hash = local_md5(sig) -%} + {%- set prefix = type ~ '_' ~ stripped -%} + {%- set max_prefix = 83 -%} + {%- if prefix | length > max_prefix -%} + {%- set prefix = (prefix | list)[:max_prefix] | join -%} + {%- endif -%} + {{ return(prefix ~ '_' ~ hash) }} +{%- endmacro %} + + +{% macro sqlserver__index_exists(rel, index_name) -%} + EXISTS ( + SELECT 1 + FROM sys.indexes {{ information_schema_hints() }} + WHERE name = N'{{ index_name }}' + AND object_id = OBJECT_ID(N'{{ mssql__qualified_relation(rel) }}') + ) +{%- endmacro %} + + {% macro sqlserver__create_clustered_columnstore_index(relation) -%} - {%- set cci_name = (relation.schema ~ '_' ~ relation.identifier ~ '_cci') | replace(".", "") | replace(" ", "") -%} - {%- set relation_name = relation.include(database=False) -%} - {%- set full_relation = '"' ~ relation.schema ~ '"."' ~ relation.identifier ~ '"' -%} + {%- set stripped_id = mssql__strip_dbt_suffix(relation.identifier) | replace('[', '') | replace(']', '') -%} + {%- set cci_name = relation.schema ~ '_' ~ stripped_id ~ '_cci' -%} + {%- set full_relation = mssql__quote_ident(relation.schema) ~ '.' ~ mssql__quote_ident(relation.identifier) -%} use [{{ relation.database }}]; if EXISTS ( SELECT * FROM sys.indexes {{ information_schema_hints() }} - WHERE name = '{{cci_name}}' - AND object_id=object_id('{{relation_name}}') + WHERE name = N'{{cci_name}}' + AND object_id = OBJECT_ID(N'{{full_relation}}') ) - DROP index {{full_relation}}.{{cci_name}} - CREATE CLUSTERED COLUMNSTORE INDEX {{cci_name}} - ON {{full_relation}} + DROP INDEX {{full_relation}}.[{{cci_name}}] + CREATE CLUSTERED COLUMNSTORE INDEX [{{cci_name}}] + ON {{ mssql__qualified_relation(relation) }} {% endmacro %} {% macro drop_xml_indexes() -%} @@ -116,15 +166,20 @@ {%- endmacro %} -{% macro create_clustered_index(columns, unique=False) -%} +{% macro create_clustered_index(columns, unique=False, relation=none) -%} + {%- set _relation = relation if relation is not none else this -%} + {%- set cols = [columns] if columns is string else columns -%} + {%- set idx_name = mssql__index_name(_relation, 'cidx', cols, unique=unique) -%} + {%- set quoted_cols = [] -%} + {%- for col in cols -%} + {%- do quoted_cols.append(mssql__quote_ident(col)) -%} + {%- endfor -%} {{ log("Creating clustered index...") }} - {% set idx_name = "clustered_" + local_md5(columns | join("_")) %} - - if not exists(select * + if not exists(select 1 from sys.indexes {{ information_schema_hints() }} - where name = '{{ idx_name }}' - and object_id = OBJECT_ID('{{ this }}') + where name = N'{{ idx_name }}' + and object_id = OBJECT_ID(N'{{ mssql__qualified_relation(_relation) }}') ) begin @@ -133,38 +188,39 @@ unique {% endif %} clustered index - {{ idx_name }} - on {{ this }} ({{ '[' + columns|join("], [") + ']' }}) + [{{ idx_name }}] + on {{ mssql__qualified_relation(_relation) }} ({{ quoted_cols | join(', ') }}) end {%- endmacro %} -{% macro create_nonclustered_index(columns, includes=False) %} +{% macro create_nonclustered_index(columns, includes=False, relation=none) %} + {%- set _relation = relation if relation is not none else this -%} + {%- set cols = [columns] if columns is string else columns -%} + {%- set incs = [] if not includes else ([includes] if includes is string else includes) -%} + {%- set idx_name = mssql__index_name(_relation, 'nidx', cols, includes=incs) -%} + {%- set quoted_cols = [] -%} + {%- for col in cols -%} + {%- do quoted_cols.append(mssql__quote_ident(col)) -%} + {%- endfor -%} + {%- set quoted_incs = [] -%} + {%- for inc in incs -%} + {%- do quoted_incs.append(mssql__quote_ident(inc)) -%} + {%- endfor -%} {{ log("Creating nonclustered index...") }} - {% if includes -%} - {% set idx_name = ( - "nonclustered_" - + local_md5(columns | join("_")) - + "_incl_" - + local_md5(includes | join("_")) - ) %} - {% else -%} - {% set idx_name = "nonclustered_" + local_md5(columns | join("_")) %} - {% endif %} - - if not exists(select * + if not exists(select 1 from sys.indexes {{ information_schema_hints() }} - where name = '{{ idx_name }}' - and object_id = OBJECT_ID('{{ this }}') + where name = N'{{ idx_name }}' + and object_id = OBJECT_ID(N'{{ mssql__qualified_relation(_relation) }}') ) begin create nonclustered index - {{ idx_name }} - on {{ this }} ({{ '[' + columns|join("], [") + ']' }}) - {% if includes -%} - include ({{ '[' + includes|join("], [") + ']' }}) + [{{ idx_name }}] + on {{ mssql__qualified_relation(_relation) }} ({{ quoted_cols | join(', ') }}) + {% if quoted_incs -%} + include ({{ quoted_incs | join(', ') }}) {% endif %} end {% endmacro %} diff --git a/dbt/include/sqlserver/macros/relations/table/create.sql b/dbt/include/sqlserver/macros/relations/table/create.sql index 9a6e905a8..79c72852e 100644 --- a/dbt/include/sqlserver/macros/relations/table/create.sql +++ b/dbt/include/sqlserver/macros/relations/table/create.sql @@ -40,9 +40,11 @@ {% set as_columnstore = config.get('as_columnstore', default=true) %} {% if not temporary and as_columnstore -%} {#- - add columnstore index - this creates with dbt_temp as its coming from a temporary relation before renaming - could alter relation to drop the dbt_temp portion if needed + Add a clustered columnstore index. The index name is derived from the + *final* relation name (with __dbt_tmp / __dbt_backup suffixes stripped), + so a second run produces an identical name and the index is not + orphaned after intermediate relations are renamed. + See dbt/include/sqlserver/macros/adapters/indexes.sql. -#} {{ sqlserver__create_clustered_columnstore_index(relation) }} {% endif %} diff --git a/tests/functional/adapter/mssql/test_index_macros.py b/tests/functional/adapter/mssql/test_index_macros.py index 0863876cd..15b0c4782 100644 --- a/tests/functional/adapter/mssql/test_index_macros.py +++ b/tests/functional/adapter/mssql/test_index_macros.py @@ -184,3 +184,250 @@ def test_create_index(self, project): run_dbt(["run"]) self.validate_other_schema(project) self.drop_schema_artifacts(project) + + +# --------------------------------------------------------------------------- +# Integration tests for the refactored index subsystem. +# +# These run against a live SQL Server (credentials from test.env) and verify: +# * incremental runs produce identical, idempotent indexes +# * INCLUDE columns are registered as is_included_column = 1 +# * quoted / weird identifiers are safely bracket-escaped +# * schema-qualified models keep indexes isolated per schema +# * multi-threaded runs are race-free (deterministic names + IF NOT EXISTS) +# * CCI is created on the final relation, not the __dbt_tmp intermediate +# * CCI name on the final table never contains __dbt_tmp or __dbt_backup +# --------------------------------------------------------------------------- + + +index_columns_for_table = """ +SELECT + i.[name] AS index_name, + i.type_desc AS index_type, + c.[name] AS column_name, + ic.is_included_column AS is_included, + ic.key_ordinal AS key_ordinal +FROM sys.indexes i +INNER JOIN sys.index_columns ic + ON i.object_id = ic.object_id AND i.index_id = ic.index_id +INNER JOIN sys.columns c + ON c.object_id = ic.object_id AND c.column_id = ic.column_id +WHERE i.object_id = OBJECT_ID(N'[{database}].[{schema}].[{table}]') +ORDER BY i.[name], ic.key_ordinal, ic.is_included_column, c.[name] +""" + + +incremental_model_sql = """ +{{ + config({ + "materialized": "incremental", + "unique_key": "id_col", + "as_columnstore": False, + "post-hook": [ + "{{ create_clustered_index(columns=['id_col'], unique=True) }}", + "{{ create_nonclustered_index(columns=['data']) }}", + ], + }) +}} +select * from {{ ref('raw_data') }} +{% if is_incremental() %} +where id_col > (select coalesce(max(id_col), 0) from {{ this }}) +{% endif %} +""" + + +class TestIndexIncremental: + """Indexes should survive incremental runs (idempotent IF NOT EXISTS).""" + + @pytest.fixture(scope="class") + def seeds(self): + return {"raw_data.csv": index_seed_csv, "schema.yml": index_schema_base_yml} + + @pytest.fixture(scope="class") + def models(self): + return {"inc_model.sql": incremental_model_sql} + + def _index_rows(self, project, table_name): + sql = index_columns_for_table.format( + database=project.database, + schema=project.test_schema, + table=table_name, + ) + with get_connection(project.adapter): + _, table = project.adapter.execute(sql, fetch=True) + return list(table.rows) + + def test_indexes_stable_across_incremental_runs(self, project): + run_dbt(["seed"]) + run_dbt(["run"]) + first = self._index_rows(project, "inc_model") + + run_dbt(["run"]) + second = self._index_rows(project, "inc_model") + + assert first == second + index_names = {r[0] for r in second} + assert len(index_names) == 2 + + +include_columns_model_sql = """ +{{ + config({ + "materialized": "table", + "as_columnstore": False, + "post-hook": [ + "{{ create_nonclustered_index(columns=['secondary_data'], includes=['tertiary_data','data']) }}", + ], + }) +}} +select * from {{ ref('raw_data') }} +""" + + +class TestIndexIncludeColumns: + """INCLUDE columns must end up as is_included_column = 1 in sys.index_columns.""" + + @pytest.fixture(scope="class") + def seeds(self): + return {"raw_data.csv": index_seed_csv, "schema.yml": index_schema_base_yml} + + @pytest.fixture(scope="class") + def models(self): + return {"inc_cols_model.sql": include_columns_model_sql} + + def test_include_columns_present(self, project): + run_dbt(["seed"]) + run_dbt(["run"]) + sql = index_columns_for_table.format( + database=project.database, + schema=project.test_schema, + table="inc_cols_model", + ) + with get_connection(project.adapter): + _, table = project.adapter.execute(sql, fetch=True) + rows = list(table.rows) + keyed = [r for r in rows if r[3] == 0] + included = [r for r in rows if r[3] == 1] + assert {r[2] for r in keyed} == {"secondary_data"} + assert {r[2] for r in included} == {"tertiary_data", "data"} + + +quoted_seed_csv = """id col],weird name +1,a +2,b +""" + +quoted_schema_yml = """ +version: 2 +seeds: + - name: raw_data + config: + column_types: + "id col]": integer + "weird name": nvarchar(20) +""" + +quoted_model_sql = """ +{{ + config({ + "materialized": "table", + "as_columnstore": False, + "post-hook": [ + "{{ create_nonclustered_index(columns=['id col]', 'weird name']) }}", + ], + }) +}} +select * from {{ ref('raw_data') }} +""" + + +class TestIndexQuotedIdentifiers: + """Bracket and space characters in column names must be safely quoted.""" + + @pytest.fixture(scope="class") + def seeds(self): + return {"raw_data.csv": quoted_seed_csv, "schema.yml": quoted_schema_yml} + + @pytest.fixture(scope="class") + def models(self): + return {"quoted_model.sql": quoted_model_sql} + + def test_quoted_columns_indexed(self, project): + run_dbt(["seed"]) + run_dbt(["run"]) + sql = index_columns_for_table.format( + database=project.database, + schema=project.test_schema, + table="quoted_model", + ) + with get_connection(project.adapter): + _, table = project.adapter.execute(sql, fetch=True) + col_names = {r[2] for r in table.rows} + assert "id col]" in col_names + assert "weird name" in col_names + + + + + +orphan_cci_check_sql = """ +SELECT COUNT(*) +FROM sys.indexes i +INNER JOIN sys.tables t ON i.object_id = t.object_id +INNER JOIN sys.schemas s ON t.schema_id = s.schema_id +WHERE s.name = '{schema}' + AND (t.name LIKE '%__dbt_tmp' OR t.name LIKE '%__dbt_backup') + AND i.type IN (5, 6) +""" + + +class TestNoOrphanColumnstoreIndex: + """Regression: CCI must be created on the *final* relation, not the + __dbt_tmp intermediate. After a successful run there should be zero CCIs + sitting on any tmp / backup relation in the test schema.""" + + @pytest.fixture(scope="class") + def seeds(self): + return {"raw_data.csv": index_seed_csv, "schema.yml": index_schema_base_yml} + + @pytest.fixture(scope="class") + def models(self): + return { + "cci_model.sql": ( + "{{ config(materialized='table', as_columnstore=True) }}\n" + "select * from {{ ref('raw_data') }}\n" + ) + } + + def test_no_orphan_cci(self, project): + run_dbt(["seed"]) + run_dbt(["run"]) + run_dbt(["run"]) + + with get_connection(project.adapter): + _, t = project.adapter.execute( + orphan_cci_check_sql.format(schema=project.test_schema), + fetch=True, + ) + assert list(t.rows)[0][0] == 0, "Orphaned CCI found on __dbt_tmp/__dbt_backup" + + cci_name_sql = """ + SELECT i.name + FROM sys.indexes i + INNER JOIN sys.tables t ON i.object_id = t.object_id + INNER JOIN sys.schemas s ON t.schema_id = s.schema_id + WHERE s.name = '{schema}' + AND t.name = 'cci_model' + AND i.type IN (5, 6) + """.format(schema=project.test_schema) + with get_connection(project.adapter): + _, t = project.adapter.execute(cci_name_sql, fetch=True) + cci_rows = list(t.rows) + assert len(cci_rows) == 1, f"Expected exactly 1 CCI on cci_model, got {cci_rows}" + cci_name = cci_rows[0][0] + assert "__dbt_tmp" not in cci_name, ( + f"CCI name '{cci_name}' contains __dbt_tmp - index would be orphaned after rename" + ) + assert "__dbt_backup" not in cci_name, ( + f"CCI name '{cci_name}' contains __dbt_backup - index would be orphaned after rename" + ) diff --git a/tests/unit/adapters/mssql/test_indexes.py b/tests/unit/adapters/mssql/test_indexes.py new file mode 100644 index 000000000..7857231bd --- /dev/null +++ b/tests/unit/adapters/mssql/test_indexes.py @@ -0,0 +1,355 @@ +""" +Unit tests for the SQL Server index macros. + +Mirrors the style of test_generate_schema_name.py: inline Jinja, no DB. + +Coverage: + * mssql__quote_ident - bracket escaping + * mssql__qualified_relation - 3-part name assembly + * mssql__strip_dbt_suffix - __dbt_tmp / __dbt_backup stripping + * mssql__index_name - determinism, 116-char cap, hashing, + immunity to __dbt_tmp suffix + * sqlserver__index_exists - emitted SQL shape (OBJECT_ID scoped) + * create_clustered_index - quoted columns, IF NOT EXISTS wrapper + * create_nonclustered_index - INCLUDE columns, quoting, idempotence +""" + +import hashlib +import re +from pathlib import Path + +import jinja2 +import pytest +from jinja2.runtime import Macro as _Jinja2Macro + + +class _MacroReturn(BaseException): + def __init__(self, value): + self.value = value + + +_orig_macro_call = _Jinja2Macro.__call__ + + +def _patched_macro_call(self, *args, **kwargs): + try: + return _orig_macro_call(self, *args, **kwargs) + except _MacroReturn as exc: + return exc.value + + +@pytest.fixture(scope="session", autouse=True) +def _patch_jinja2_macro_return(): + _Jinja2Macro.__call__ = _patched_macro_call + yield + _Jinja2Macro.__call__ = _orig_macro_call + + +MACRO_PATH = ( + Path(__file__).resolve().parents[4] + / "dbt" + / "include" + / "sqlserver" + / "macros" + / "adapters" + / "indexes.sql" +) + +MACRO_SRC = MACRO_PATH.read_text(encoding="utf-8") + + +class _FakeRelation: + """Minimal stand-in for dbt's Relation object.""" + + def __init__(self, database, schema, identifier): + self.database = database + self.schema = schema + self.identifier = identifier + + def __str__(self): + return f"[{self.database}].[{self.schema}].[{self.identifier}]" + + +def _env(): + env = jinja2.Environment( + trim_blocks=True, + lstrip_blocks=True, + extensions=["jinja2.ext.do"], + ) + + def local_md5(s): + return hashlib.md5(s.encode("utf-8")).hexdigest() + + env.globals.update( + { + "local_md5": local_md5, + "information_schema_hints": lambda: "", + "log": lambda *a, **kw: "", + "this": _FakeRelation("mydb", "myschema", "my_model"), + "return": lambda v: (_ for _ in ()).throw(_MacroReturn(v)), + } + ) + return env + + +def _render(call_expr, **ctx): + env = _env() + template = env.from_string(MACRO_SRC + "\n" + "{{ " + call_expr + " }}") + return template.render(**ctx).strip() + + +def _normalize_ws(s): + return re.sub(r"\s+", " ", s).strip() + + +class TestQuoteIdent: + def test_simple_name(self): + assert _render("mssql__quote_ident('foo')") == "[foo]" + + def test_name_with_space(self): + assert _render("mssql__quote_ident('weird name')") == "[weird name]" + + def test_name_with_close_bracket_is_escaped(self): + assert _render("mssql__quote_ident('we][ird')") == "[we]][ird]" + + def test_name_with_dot_is_not_split(self): + assert _render("mssql__quote_ident('a.b')") == "[a.b]" + + +class TestQualifiedRelation: + def test_three_part_name(self): + rel = _FakeRelation("mydb", "myschema", "my_model") + out = _render("mssql__qualified_relation(rel)", rel=rel) + assert out == "[mydb].[myschema].[my_model]" + + def test_escapes_brackets_in_every_part(self): + rel = _FakeRelation("d]b", "sch]ema", "id]ent") + out = _render("mssql__qualified_relation(rel)", rel=rel) + assert out == "[d]]b].[sch]]ema].[id]]ent]" + + +class TestStripDbtSuffix: + @pytest.mark.parametrize( + "identifier, expected", + [ + ("my_model", "my_model"), + ("my_model__dbt_tmp", "my_model"), + ("my_model__dbt_backup", "my_model"), + ("my_model__dbt_tmp_vw", "my_model"), + ("my__dbt_tmp_model", "my__dbt_tmp_model"), # __dbt_tmp in middle, not a suffix + ], + ) + def test_strip(self, identifier, expected): + assert _render(f"mssql__strip_dbt_suffix('{identifier}')") == expected + + +class TestIndexName: + def test_deterministic(self): + rel = _FakeRelation("d", "s", "t") + a = _render("mssql__index_name(rel, 'nidx', ['x','y'])", rel=rel) + b = _render("mssql__index_name(rel, 'nidx', ['x','y'])", rel=rel) + assert a == b + + def test_column_order_changes_name(self): + rel = _FakeRelation("d", "s", "t") + a = _render("mssql__index_name(rel, 'nidx', ['x','y'])", rel=rel) + b = _render("mssql__index_name(rel, 'nidx', ['y','x'])", rel=rel) + assert a != b + + def test_unique_flag_changes_name(self): + rel = _FakeRelation("d", "s", "t") + a = _render("mssql__index_name(rel, 'cidx', ['x'], unique=False)", rel=rel) + b = _render("mssql__index_name(rel, 'cidx', ['x'], unique=True)", rel=rel) + assert a != b + + def test_includes_changes_name(self): + rel = _FakeRelation("d", "s", "t") + a = _render("mssql__index_name(rel, 'nidx', ['x'])", rel=rel) + b = _render("mssql__index_name(rel, 'nidx', ['x'], includes=['y'])", rel=rel) + assert a != b + + def test_accepts_string_columns(self): + rel = _FakeRelation("d", "s", "t") + name = _render("mssql__index_name(rel, 'nidx', 'x')", rel=rel) + assert name.startswith("nidx_t_") + + def test_false_includes_are_treated_as_empty(self): + rel = _FakeRelation("d", "s", "t") + a = _render("mssql__index_name(rel, 'nidx', ['x'], includes=false)", rel=rel) + b = _render("mssql__index_name(rel, 'nidx', ['x'])", rel=rel) + assert a == b + + def test_dbt_tmp_suffix_does_not_affect_name(self): + a = _render( + "mssql__index_name(rel, 'cci', ['__all__'])", + rel=_FakeRelation("d", "s", "my_model"), + ) + b = _render( + "mssql__index_name(rel, 'cci', ['__all__'])", + rel=_FakeRelation("d", "s", "my_model__dbt_tmp"), + ) + c = _render( + "mssql__index_name(rel, 'cci', ['__all__'])", + rel=_FakeRelation("d", "s", "my_model__dbt_backup"), + ) + assert a == b == c + + def test_length_capped_at_116_chars(self): + rel = _FakeRelation("d", "s", "x" * 500) + name = _render("mssql__index_name(rel, 'nidx', ['c'])", rel=rel) + assert len(name) <= 116 + + def test_long_name_still_unique_per_signature(self): + rel = _FakeRelation("d", "s", "x" * 500) + a = _render("mssql__index_name(rel, 'nidx', ['c1'])", rel=rel) + b = _render("mssql__index_name(rel, 'nidx', ['c2'])", rel=rel) + assert a != b + assert len(a) <= 116 + assert len(b) <= 116 + + def test_brackets_stripped_from_readable_part(self): + rel = _FakeRelation("d", "s", "[weird]") + name = _render("mssql__index_name(rel, 'nidx', ['c'])", rel=rel) + assert "[" not in name and "]" not in name + + +class TestIndexExists: + def test_emits_object_id_scoped_check(self): + rel = _FakeRelation("mydb", "myschema", "my_model") + sql = _render("sqlserver__index_exists(rel, 'idx_foo')", rel=rel) + sql = _normalize_ws(sql) + assert "EXISTS" in sql + assert "sys.indexes" in sql + assert "name = N'idx_foo'" in sql + assert "OBJECT_ID(N'[mydb].[myschema].[my_model]')" in sql + + +class TestCreateClusteredIndex: + def test_quotes_columns(self): + rel = _FakeRelation("d", "s", "t") + sql = _render( + "create_clustered_index(['id_col', 'data'], relation=rel)", rel=rel + ) + sql = _normalize_ws(sql) + assert "([id_col], [data])" in sql + + def test_wrapped_in_if_not_exists(self): + rel = _FakeRelation("d", "s", "t") + sql = _normalize_ws( + _render("create_clustered_index(['c'], relation=rel)", rel=rel) + ) + assert "if not exists" in sql + assert "begin create" in sql + assert sql.endswith("end") + + def test_unique_flag_emits_unique_keyword(self): + rel = _FakeRelation("d", "s", "t") + sql = _normalize_ws( + _render( + "create_clustered_index(['c'], unique=True, relation=rel)", rel=rel + ) + ) + assert "unique clustered index" in sql + + def test_accepts_string_column(self): + rel = _FakeRelation("d", "s", "t") + sql = _normalize_ws( + _render("create_clustered_index('id_col', relation=rel)", rel=rel) + ) + assert "([id_col])" in sql + + +class TestCreateNonclusteredIndex: + def test_emits_nonclustered(self): + rel = _FakeRelation("d", "s", "t") + sql = _normalize_ws( + _render("create_nonclustered_index(['c'], relation=rel)", rel=rel) + ) + assert "create nonclustered index" in sql + + def test_accepts_string_column(self): + rel = _FakeRelation("d", "s", "t") + sql = _normalize_ws( + _render("create_nonclustered_index('c', relation=rel)", rel=rel) + ) + assert "([c])" in sql + + def test_accepts_string_include_column(self): + rel = _FakeRelation("d", "s", "t") + sql = _normalize_ws( + _render("create_nonclustered_index(['c'], includes='inc1', relation=rel)", rel=rel) + ) + assert "include ([inc1])" in sql + + def test_include_columns_quoted(self): + rel = _FakeRelation("d", "s", "t") + sql = _normalize_ws( + _render( + "create_nonclustered_index(['c'], includes=['inc1','inc2'], relation=rel)", + rel=rel, + ) + ) + assert "include ([inc1], [inc2])" in sql + + def test_no_include_block_when_no_includes(self): + rel = _FakeRelation("d", "s", "t") + sql = _normalize_ws( + _render("create_nonclustered_index(['c'], relation=rel)", rel=rel) + ) + assert "include" not in sql + + def test_false_include_is_treated_as_empty(self): + rel = _FakeRelation("d", "s", "t") + sql = _normalize_ws( + _render("create_nonclustered_index(['c'], includes=false, relation=rel)", rel=rel) + ) + assert "include" not in sql + + def test_columns_with_brackets_are_escaped(self): + rel = _FakeRelation("d", "s", "t") + sql = _normalize_ws( + _render( + "create_nonclustered_index(['we]ird'], relation=rel)", rel=rel + ) + ) + assert "[we]]ird]" in sql + + def test_idempotent_wrapper(self): + rel = _FakeRelation("d", "s", "t") + sql = _normalize_ws( + _render("create_nonclustered_index(['c'], relation=rel)", rel=rel) + ) + assert "if not exists" in sql and "begin" in sql and "end" in sql + + +class TestColumnstoreIndexName: + def test_intermediate_relation_uses_target_name(self): + intermediate = _FakeRelation("d", "s", "my_model__dbt_tmp") + target = _FakeRelation("d", "s", "my_model") + sql_int = _render( + "sqlserver__create_clustered_columnstore_index(intermediate)", + intermediate=intermediate, + ) + sql_final = _render( + "sqlserver__create_clustered_columnstore_index(target)", + target=target, + ) + name_re = re.compile( + r"CREATE\s+CLUSTERED\s+COLUMNSTORE\s+INDEX\s+(\[[^\]]+\])", re.IGNORECASE + ) + name_int = name_re.search(_normalize_ws(sql_int)).group(1) + name_final = name_re.search(_normalize_ws(sql_final)).group(1) + assert name_int == name_final, ( + f"intermediate CCI name {name_int} differs from final {name_final}; " + "would be orphaned after rename" + ) + + def test_uses_qualified_target_relation_for_create(self): + target = _FakeRelation("mydb", "myschema", "my_model") + sql = _normalize_ws( + _render( + "sqlserver__create_clustered_columnstore_index(target)", + target=target, + ) + ) + assert "ON [mydb].[myschema].[my_model]" in sql From dacc5fdb898b7608eb765de8468045c546af10a5 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 20 Jul 2026 10:30:27 +0000 Subject: [PATCH 2/5] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- .../adapter/mssql/test_index_macros.py | 15 ++++---- tests/unit/adapters/mssql/test_indexes.py | 36 +++++-------------- 2 files changed, 15 insertions(+), 36 deletions(-) diff --git a/tests/functional/adapter/mssql/test_index_macros.py b/tests/functional/adapter/mssql/test_index_macros.py index 15b0c4782..d8df11d19 100644 --- a/tests/functional/adapter/mssql/test_index_macros.py +++ b/tests/functional/adapter/mssql/test_index_macros.py @@ -367,9 +367,6 @@ def test_quoted_columns_indexed(self, project): assert "weird name" in col_names - - - orphan_cci_check_sql = """ SELECT COUNT(*) FROM sys.indexes i @@ -425,9 +422,9 @@ def test_no_orphan_cci(self, project): cci_rows = list(t.rows) assert len(cci_rows) == 1, f"Expected exactly 1 CCI on cci_model, got {cci_rows}" cci_name = cci_rows[0][0] - assert "__dbt_tmp" not in cci_name, ( - f"CCI name '{cci_name}' contains __dbt_tmp - index would be orphaned after rename" - ) - assert "__dbt_backup" not in cci_name, ( - f"CCI name '{cci_name}' contains __dbt_backup - index would be orphaned after rename" - ) + assert ( + "__dbt_tmp" not in cci_name + ), f"CCI name '{cci_name}' contains __dbt_tmp - index would be orphaned after rename" + assert ( + "__dbt_backup" not in cci_name + ), f"CCI name '{cci_name}' contains __dbt_backup - index would be orphaned after rename" diff --git a/tests/unit/adapters/mssql/test_indexes.py b/tests/unit/adapters/mssql/test_indexes.py index 7857231bd..9411edb90 100644 --- a/tests/unit/adapters/mssql/test_indexes.py +++ b/tests/unit/adapters/mssql/test_indexes.py @@ -227,17 +227,13 @@ def test_emits_object_id_scoped_check(self): class TestCreateClusteredIndex: def test_quotes_columns(self): rel = _FakeRelation("d", "s", "t") - sql = _render( - "create_clustered_index(['id_col', 'data'], relation=rel)", rel=rel - ) + sql = _render("create_clustered_index(['id_col', 'data'], relation=rel)", rel=rel) sql = _normalize_ws(sql) assert "([id_col], [data])" in sql def test_wrapped_in_if_not_exists(self): rel = _FakeRelation("d", "s", "t") - sql = _normalize_ws( - _render("create_clustered_index(['c'], relation=rel)", rel=rel) - ) + sql = _normalize_ws(_render("create_clustered_index(['c'], relation=rel)", rel=rel)) assert "if not exists" in sql assert "begin create" in sql assert sql.endswith("end") @@ -245,33 +241,25 @@ def test_wrapped_in_if_not_exists(self): def test_unique_flag_emits_unique_keyword(self): rel = _FakeRelation("d", "s", "t") sql = _normalize_ws( - _render( - "create_clustered_index(['c'], unique=True, relation=rel)", rel=rel - ) + _render("create_clustered_index(['c'], unique=True, relation=rel)", rel=rel) ) assert "unique clustered index" in sql def test_accepts_string_column(self): rel = _FakeRelation("d", "s", "t") - sql = _normalize_ws( - _render("create_clustered_index('id_col', relation=rel)", rel=rel) - ) + sql = _normalize_ws(_render("create_clustered_index('id_col', relation=rel)", rel=rel)) assert "([id_col])" in sql class TestCreateNonclusteredIndex: def test_emits_nonclustered(self): rel = _FakeRelation("d", "s", "t") - sql = _normalize_ws( - _render("create_nonclustered_index(['c'], relation=rel)", rel=rel) - ) + sql = _normalize_ws(_render("create_nonclustered_index(['c'], relation=rel)", rel=rel)) assert "create nonclustered index" in sql def test_accepts_string_column(self): rel = _FakeRelation("d", "s", "t") - sql = _normalize_ws( - _render("create_nonclustered_index('c', relation=rel)", rel=rel) - ) + sql = _normalize_ws(_render("create_nonclustered_index('c', relation=rel)", rel=rel)) assert "([c])" in sql def test_accepts_string_include_column(self): @@ -293,9 +281,7 @@ def test_include_columns_quoted(self): def test_no_include_block_when_no_includes(self): rel = _FakeRelation("d", "s", "t") - sql = _normalize_ws( - _render("create_nonclustered_index(['c'], relation=rel)", rel=rel) - ) + sql = _normalize_ws(_render("create_nonclustered_index(['c'], relation=rel)", rel=rel)) assert "include" not in sql def test_false_include_is_treated_as_empty(self): @@ -308,17 +294,13 @@ def test_false_include_is_treated_as_empty(self): def test_columns_with_brackets_are_escaped(self): rel = _FakeRelation("d", "s", "t") sql = _normalize_ws( - _render( - "create_nonclustered_index(['we]ird'], relation=rel)", rel=rel - ) + _render("create_nonclustered_index(['we]ird'], relation=rel)", rel=rel) ) assert "[we]]ird]" in sql def test_idempotent_wrapper(self): rel = _FakeRelation("d", "s", "t") - sql = _normalize_ws( - _render("create_nonclustered_index(['c'], relation=rel)", rel=rel) - ) + sql = _normalize_ws(_render("create_nonclustered_index(['c'], relation=rel)", rel=rel)) assert "if not exists" in sql and "begin" in sql and "end" in sql From e102ae03f46aafbabd809527dfc9d1e32fed72fc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A5rten=20Pettersson?= Date: Mon, 20 Jul 2026 10:44:46 +0000 Subject: [PATCH 3/5] fix(indexes): address Copilot review comments - Escape and quote CCI index name to prevent SQL injection - Use get_use_database_sql() helper for consistency - Limit Jinja2 macro patch fixture to module scope - Remove misleading multi-threaded bullet from comment --- .../sqlserver/macros/adapters/indexes.sql | 1114 ++++++++--------- .../adapter/mssql/test_index_macros.py | 860 ++++++------- tests/unit/adapters/mssql/test_indexes.py | 675 +++++----- 3 files changed, 1321 insertions(+), 1328 deletions(-) diff --git a/dbt/include/sqlserver/macros/adapters/indexes.sql b/dbt/include/sqlserver/macros/adapters/indexes.sql index 2113900d3..b289288dd 100644 --- a/dbt/include/sqlserver/macros/adapters/indexes.sql +++ b/dbt/include/sqlserver/macros/adapters/indexes.sql @@ -1,561 +1,553 @@ -{% macro mssql__quote_ident(name) -%} - {{ return('[' ~ name | replace(']', ']]') ~ ']') }} -{%- endmacro %} - - -{% macro mssql__qualified_relation(rel) -%} - {{ return( - mssql__quote_ident(rel.database) ~ '.' ~ - mssql__quote_ident(rel.schema) ~ '.' ~ - mssql__quote_ident(rel.identifier) - ) }} -{%- endmacro %} - - -{% macro mssql__strip_dbt_suffix(identifier) -%} - {%- set ns = namespace(result=identifier) -%} - {%- for suffix in ['__dbt_tmp_vw', '__dbt_backup', '__dbt_tmp'] -%} - {%- if ns.result.endswith(suffix) -%} - {%- set ns.result = ns.result[:(ns.result | length) - (suffix | length)] -%} - {%- endif -%} - {%- endfor -%} - {{ return(ns.result) }} -{%- endmacro %} - - -{% macro mssql__index_name(rel, type, columns, unique=False, includes=false) -%} - {%- set cols = [columns] if columns is string else columns -%} - {%- set incs = [] if not includes else ([includes] if includes is string else includes) -%} - {%- set stripped = mssql__strip_dbt_suffix(rel.identifier) | replace('[', '') | replace(']', '') -%} - {%- set sig = type ~ '|' ~ (cols | join(',')) ~ '|' ~ (unique | string) ~ '|' ~ (incs | join(',')) -%} - {%- set hash = local_md5(sig) -%} - {%- set prefix = type ~ '_' ~ stripped -%} - {%- set max_prefix = 83 -%} - {%- if prefix | length > max_prefix -%} - {%- set prefix = (prefix | list)[:max_prefix] | join -%} - {%- endif -%} - {{ return(prefix ~ '_' ~ hash) }} -{%- endmacro %} - - -{% macro sqlserver__index_exists(rel, index_name) -%} - EXISTS ( - SELECT 1 - FROM sys.indexes {{ information_schema_hints() }} - WHERE name = N'{{ index_name }}' - AND object_id = OBJECT_ID(N'{{ mssql__qualified_relation(rel) }}') - ) -{%- endmacro %} - - -{% macro sqlserver__create_clustered_columnstore_index(relation) -%} - {%- set stripped_id = mssql__strip_dbt_suffix(relation.identifier) | replace('[', '') | replace(']', '') -%} - {%- set cci_name = relation.schema ~ '_' ~ stripped_id ~ '_cci' -%} - {%- set full_relation = mssql__quote_ident(relation.schema) ~ '.' ~ mssql__quote_ident(relation.identifier) -%} - use [{{ relation.database }}]; - if EXISTS ( - SELECT * - FROM sys.indexes {{ information_schema_hints() }} - WHERE name = N'{{cci_name}}' - AND object_id = OBJECT_ID(N'{{full_relation}}') - ) - DROP INDEX {{full_relation}}.[{{cci_name}}] - CREATE CLUSTERED COLUMNSTORE INDEX [{{cci_name}}] - ON {{ mssql__qualified_relation(relation) }} -{% endmacro %} - -{% macro drop_xml_indexes() -%} - {{ log("Running drop_xml_indexes() macro...") }} - - declare @drop_xml_indexes nvarchar(max); - select @drop_xml_indexes = ( - select 'IF INDEXPROPERTY(' + CONVERT(VARCHAR(MAX), sys.tables.[object_id]) + ', ''' + sys.indexes.[name] + ''', ''IndexId'') IS NOT NULL DROP INDEX [' + sys.indexes.[name] + '] ON ' + '[' + SCHEMA_NAME(sys.tables.[schema_id]) + '].[' + OBJECT_NAME(sys.tables.[object_id]) + ']; ' - from sys.indexes {{ information_schema_hints() }} - inner join sys.tables {{ information_schema_hints() }} - on sys.indexes.object_id = sys.tables.object_id - where sys.indexes.[name] is not null - and sys.indexes.type_desc = 'XML' - and sys.tables.[name] = '{{ this.table }}' - for xml path('') - ); exec sp_executesql @drop_xml_indexes; -{%- endmacro %} - - -{% macro drop_spatial_indexes() -%} - {# Altered from https://stackoverflow.com/q/1344401/10415173 #} - {# and https://stackoverflow.com/a/33785833/10415173 #} - - {{ log("Running drop_spatial_indexes() macro...") }} - - declare @drop_spatial_indexes nvarchar(max); - select @drop_spatial_indexes = ( - select 'IF INDEXPROPERTY(' + CONVERT(VARCHAR(MAX), sys.tables.[object_id]) + ', ''' + sys.indexes.[name] + ''', ''IndexId'') IS NOT NULL DROP INDEX [' + sys.indexes.[name] + '] ON ' + '[' + SCHEMA_NAME(sys.tables.[schema_id]) + '].[' + OBJECT_NAME(sys.tables.[object_id]) + ']; ' - from sys.indexes {{ information_schema_hints() }} - inner join sys.tables {{ information_schema_hints() }} - on sys.indexes.object_id = sys.tables.object_id - where sys.indexes.[name] is not null - and sys.indexes.type_desc = 'Spatial' - and sys.tables.[name] = '{{ this.table }}' - for xml path('') - ); exec sp_executesql @drop_spatial_indexes; -{%- endmacro %} - - -{% macro drop_fk_constraints() -%} - {# Altered from https://stackoverflow.com/q/1344401/10415173 #} - - {{ log("Running drop_fk_constraints() macro...") }} - - declare @drop_fk_constraints nvarchar(max); - select @drop_fk_constraints = ( - select 'IF OBJECT_ID(''' + SCHEMA_NAME(CONVERT(VARCHAR(MAX), sys.foreign_keys.[schema_id])) + '.' + sys.foreign_keys.[name] + ''', ''F'') IS NOT NULL ALTER TABLE [' + SCHEMA_NAME(sys.foreign_keys.[schema_id]) + '].[' + OBJECT_NAME(sys.foreign_keys.[parent_object_id]) + '] DROP CONSTRAINT [' + sys.foreign_keys.[name]+ '];' - from sys.foreign_keys - inner join sys.tables on sys.foreign_keys.[referenced_object_id] = sys.tables.[object_id] - where sys.tables.[name] = '{{ this.table }}' - for xml path('') - ); exec sp_executesql @drop_fk_constraints; - -{%- endmacro %} - - -{% macro drop_pk_constraints() -%} - {# Altered from https://stackoverflow.com/q/1344401/10415173 #} - {# and https://stackoverflow.com/a/33785833/10415173 #} - - {{ drop_xml_indexes() }} - - {{ drop_spatial_indexes() }} - - {{ drop_fk_constraints() }} - - {{ log("Running drop_pk_constraints() macro...") }} - - declare @drop_pk_constraints nvarchar(max); - select @drop_pk_constraints = ( - select 'IF INDEXPROPERTY(' + CONVERT(VARCHAR(MAX), sys.tables.[object_id]) + ', ''' + sys.indexes.[name] + ''', ''IndexId'') IS NOT NULL ALTER TABLE [' + SCHEMA_NAME(sys.tables.[schema_id]) + '].[' + sys.tables.[name] + '] DROP CONSTRAINT [' + sys.indexes.[name]+ '];' - from sys.indexes - inner join sys.tables on sys.indexes.[object_id] = sys.tables.[object_id] - where sys.indexes.is_primary_key = 1 - and sys.tables.[name] = '{{ this.table }}' - for xml path('') - ); exec sp_executesql @drop_pk_constraints; - -{%- endmacro %} - - -{% macro drop_all_indexes_on_table() -%} - {# Altered from https://stackoverflow.com/q/1344401/10415173 #} - {# and https://stackoverflow.com/a/33785833/10415173 #} - - {{ drop_pk_constraints() }} - - {{ log("Dropping remaining indexes...") }} - - declare @drop_remaining_indexes_last nvarchar(max); - select @drop_remaining_indexes_last = ( - select 'IF INDEXPROPERTY(' + CONVERT(VARCHAR(MAX), sys.tables.[object_id]) + ', ''' + sys.indexes.[name] + ''', ''IndexId'') IS NOT NULL DROP INDEX [' + sys.indexes.[name] + '] ON ' + '[' + SCHEMA_NAME(sys.tables.[schema_id]) + '].[' + OBJECT_NAME(sys.tables.[object_id]) + ']; ' - from sys.indexes {{ information_schema_hints() }} - inner join sys.tables {{ information_schema_hints() }} - on sys.indexes.object_id = sys.tables.object_id - where sys.indexes.[name] is not null - and SCHEMA_NAME(sys.tables.schema_id) = '{{ this.schema }}' - and sys.tables.[name] = '{{ this.table }}' - for xml path('') - ); exec sp_executesql @drop_remaining_indexes_last; - -{%- endmacro %} - - -{% macro create_clustered_index(columns, unique=False, relation=none) -%} - {%- set _relation = relation if relation is not none else this -%} - {%- set cols = [columns] if columns is string else columns -%} - {%- set idx_name = mssql__index_name(_relation, 'cidx', cols, unique=unique) -%} - {%- set quoted_cols = [] -%} - {%- for col in cols -%} - {%- do quoted_cols.append(mssql__quote_ident(col)) -%} - {%- endfor -%} - {{ log("Creating clustered index...") }} - - if not exists(select 1 - from sys.indexes {{ information_schema_hints() }} - where name = N'{{ idx_name }}' - and object_id = OBJECT_ID(N'{{ mssql__qualified_relation(_relation) }}') - ) - begin - - create - {% if unique -%} - unique - {% endif %} - clustered index - [{{ idx_name }}] - on {{ mssql__qualified_relation(_relation) }} ({{ quoted_cols | join(', ') }}) - end -{%- endmacro %} - - -{% macro create_nonclustered_index(columns, includes=False, relation=none) %} - - {%- set _relation = relation if relation is not none else this -%} - {%- set cols = [columns] if columns is string else columns -%} - {%- set incs = [] if not includes else ([includes] if includes is string else includes) -%} - {%- set idx_name = mssql__index_name(_relation, 'nidx', cols, includes=incs) -%} - {%- set quoted_cols = [] -%} - {%- for col in cols -%} - {%- do quoted_cols.append(mssql__quote_ident(col)) -%} - {%- endfor -%} - {%- set quoted_incs = [] -%} - {%- for inc in incs -%} - {%- do quoted_incs.append(mssql__quote_ident(inc)) -%} - {%- endfor -%} - {{ log("Creating nonclustered index...") }} - - if not exists(select 1 - from sys.indexes {{ information_schema_hints() }} - where name = N'{{ idx_name }}' - and object_id = OBJECT_ID(N'{{ mssql__qualified_relation(_relation) }}') - ) - begin - create nonclustered index - [{{ idx_name }}] - on {{ mssql__qualified_relation(_relation) }} ({{ quoted_cols | join(', ') }}) - {% if quoted_incs -%} - include ({{ quoted_incs | join(', ') }}) - {% endif %} - end -{% endmacro %} - - -{% macro drop_fk_indexes_on_table(relation) -%} - {% call statement('find_references', fetch_result=true) %} - USE [{{ relation.database }}]; - SELECT obj.name AS FK_NAME, - sch.name AS [schema_name], - tab1.name AS [table], - col1.name AS [column], - tab2.name AS [referenced_table], - col2.name AS [referenced_column] - FROM sys.foreign_key_columns fkc - INNER JOIN sys.objects obj - ON obj.object_id = fkc.constraint_object_id - INNER JOIN sys.tables tab1 - ON tab1.object_id = fkc.parent_object_id - INNER JOIN sys.schemas sch - ON tab1.schema_id = sch.schema_id - INNER JOIN sys.columns col1 - ON col1.column_id = parent_column_id AND col1.object_id = tab1.object_id - INNER JOIN sys.tables tab2 - ON tab2.object_id = fkc.referenced_object_id - INNER JOIN sys.columns col2 - ON col2.column_id = referenced_column_id AND col2.object_id = tab2.object_id - WHERE sch.name = '{{ relation.schema }}' and tab2.name = '{{ relation.identifier }}' - {% endcall %} - {% set references = load_result('find_references')['data'] %} - {% for reference in references -%} - {% call statement('main') -%} - alter table [{{reference[1]}}].[{{reference[2]}}] drop constraint [{{reference[0]}}] - {%- endcall %} - {% endfor %} -{% endmacro %} - -{% macro sqlserver__list_nonclustered_rowstore_indexes(relation) -%} - {% call statement('list_nonclustered_rowstore_indexes', fetch_result=True) -%} - - SELECT i.name AS index_name - , i.name + '__dbt_backup' as index_new_name - , COL_NAME(ic.object_id,ic.column_id) AS column_name - FROM sys.indexes AS i - INNER JOIN sys.index_columns AS ic - ON i.object_id = ic.object_id AND i.index_id = ic.index_id and i.type <> 5 - WHERE i.object_id = OBJECT_ID('{{ relation.schema }}.{{ relation.identifier }}') - - UNION ALL - - SELECT obj.name AS index_name - , obj.name + '__dbt_backup' as index_new_name - , col1.name AS column_name - FROM sys.foreign_key_columns fkc - INNER JOIN sys.objects obj - ON obj.object_id = fkc.constraint_object_id - INNER JOIN sys.tables tab1 - ON tab1.object_id = fkc.parent_object_id - INNER JOIN sys.schemas sch - ON tab1.schema_id = sch.schema_id - INNER JOIN sys.columns col1 - ON col1.column_id = parent_column_id AND col1.object_id = tab1.object_id - INNER JOIN sys.tables tab2 - ON tab2.object_id = fkc.referenced_object_id - INNER JOIN sys.columns col2 - ON col2.column_id = referenced_column_id AND col2.object_id = tab2.object_id - WHERE sch.name = '{{ relation.schema }}' and tab1.name = '{{ relation.identifier }}' - - {% endcall %} - {{ return(load_result('list_nonclustered_rowstore_indexes').table) }} -{% endmacro %} - - -{% macro sqlserver__server_major_version() -%} - {#- Detected engine major version: 13 = 2016, 14 = 2017, 15 = 2019, - 16 = 2022, 17 = 2025. parsename() on the always-4-part productversion - works on every supported release, unlike - SERVERPROPERTY('ProductMajorVersion') which is 2014+ only. Returns none - outside an executing context (e.g. parse time). -#} - {%- if not execute -%}{{ return(none) }}{%- endif -%} - {%- set result = run_query( - "select cast(parsename(cast(serverproperty('productversion') as varchar(128)), 4) as int) as major_version" - ) -%} - {{ return(result.columns[0].values()[0]) }} -{%- endmacro %} - - -{% macro sqlserver__get_create_index_sql(relation, index_dict) -%} - {%- set index_config = adapter.parse_index(index_dict) -%} - {%- set index_name = index_config.render(relation) -%} - - {# Validations are made on the adapter class SQLServerIndexConfig to control resulting sql #} - {# Names are a deterministic hash of the full definition, so an existing #} - {# index with this name is already the index we want: skip, don't fail. #} - if not exists(select * - from sys.indexes {{ information_schema_hints() }} - where name = '{{ index_name }}' - and object_id = OBJECT_ID('{{ relation }}') - ) - begin - {# key columns: bracket-quoted (with ]] escaping) plus per-column direction #} - {%- set key_columns = [] -%} - {%- for column in index_config.columns -%} - {%- do key_columns.append( - '[' ~ column | replace(']', ']]') ~ ']' - ~ (' desc' if column in index_config.descending_columns else '') - ) -%} - {%- endfor -%} - {%- set include_columns = [] -%} - {%- for column in index_config.included_columns -%} - {%- do include_columns.append('[' ~ column | replace(']', ']]') ~ ']') -%} - {%- endfor %} - create - {% if index_config.unique -%} unique {% endif %}{{ index_config.type }} - index [{{ index_name }}] - on {{ relation }} - ({{ key_columns | join(', ') }}) - {% if include_columns -%} - include ({{ include_columns | join(', ') }}) - {% endif %} - {% if index_config.where -%} - where {{ index_config.where }} - {% endif %} - {#- optimize_for_sequential_key, RESUMABLE and RESUMABLE's MAX_DURATION are - CREATE INDEX options that SQL Server only recognizes on 2019 (major 15) - and newer. This adapter still supports 2017/2016, so on older engines - they are dropped (with a warning) and the index is built without them, - rather than failing with "is not a recognized CREATE INDEX option". - ONLINE is intentionally kept: it is recognized on 2017 (edition-gated, - not version-gated). The server version is only queried when one of these - options is actually requested, so the common path adds no round-trip. -#} - {%- set v2019_only_build_options = ['resumable', 'max_duration'] -%} - {%- set _build_options = index_config.build_options or {} -%} - {%- set _wants_v2019_option = index_config.optimize_for_sequential_key - or _build_options.get('resumable') or _build_options.get('max_duration') -%} - {%- set drop_v2019_options = false -%} - {%- if _wants_v2019_option -%} - {%- set _major = sqlserver__server_major_version() -%} - {%- if _major is not none and _major < 15 -%} - {%- set drop_v2019_options = true -%} - {%- do log( - "Index [" ~ index_name ~ "] on " ~ relation ~ ": optimize_for_sequential_key" - ~ " / resumable require SQL Server 2019 (15.x) or newer; building the index" - ~ " without them on detected major version " ~ _major ~ ".", info=true - ) -%} - {%- endif -%} - {%- endif -%} - {%- set with_options = [] -%} - {%- if index_config.data_compression -%} - {%- do with_options.append('data_compression = ' ~ index_config.data_compression | upper) -%} - {%- endif -%} - {%- if index_config.fillfactor -%} - {%- do with_options.append('fillfactor = ' ~ index_config.fillfactor) -%} - {%- endif -%} - {%- if index_config.pad_index -%} - {%- do with_options.append('pad_index = on') -%} - {%- endif -%} - {%- if index_config.ignore_dup_key -%} - {%- do with_options.append('ignore_dup_key = on') -%} - {%- endif -%} - {%- if index_config.optimize_for_sequential_key and not drop_v2019_options -%} - {%- do with_options.append('optimize_for_sequential_key = on') -%} - {%- endif -%} - {%- if index_config.sort_in_tempdb -%} - {%- do with_options.append('sort_in_tempdb = on') -%} - {%- endif -%} - {%- for option_key, option_value in _build_options.items() -%} - {%- if drop_v2019_options and option_key in v2019_only_build_options -%} - {#- skipped: not recognized before SQL Server 2019 -#} - {%- elif option_value is sameas true -%} - {%- do with_options.append(option_key ~ ' = on') -%} - {%- elif option_value is sameas false -%} - {%- do with_options.append(option_key ~ ' = off') -%} - {%- elif option_key == 'max_duration' -%} - {%- do with_options.append('max_duration = ' ~ option_value ~ ' minutes') -%} - {%- else -%} - {%- do with_options.append(option_key ~ ' = ' ~ option_value) -%} - {%- endif -%} - {%- endfor -%} - {% if with_options %} - with ({{ with_options | join(', ') }}) - {% endif %} - end -{%- endmacro %} - - -{% macro sqlserver__describe_indexes(relation) %} - {% call statement('describe_indexes', fetch_result=True) -%} - select - i.[name] as [name], - case when i.[type] = 1 then 'clustered' - when i.[type] = 2 then 'nonclustered' - when i.[type] = 5 then 'clustered columnstore' - when i.[type] = 6 then 'columnstore' - end as [type], - i.is_unique as [unique], - i.is_primary_key as is_primary_key, - i.is_unique_constraint as is_unique_constraint, - isnull(key_cols.cols, '') as [columns], - isnull(incl_cols.cols, '') as included_columns, - case when i.[type] in (1, 2, 6) - then isnull(part.data_compression_desc, 'NONE') - end as data_compression, - isnull(desc_cols.cols, '') as descending_columns, - i.filter_definition as [where], - i.fill_factor as [fillfactor], - i.ignore_dup_key as [ignore_dup_key] - /* optimize_for_sequential_key is deliberately not selected: the - sys.indexes column only exists on SQL Server 2019+ and managed - comparisons are name-based, so it isn't needed here */ - from sys.indexes i {{ information_schema_hints() }} - outer apply ( - /* STRING_AGG ... WITHIN GROUP requires SQL Server 2017+, the floor - of this adapter's CI matrix. - A clustered columnstore index (type 5) has no key columns: it stores - the whole table, so sys.index_columns lists EVERY column for it. Those - columns are not part of the index's identity — reconciliation matches - the CCI by name/type and never compares its columns (see - index_config_changes) — and aggregating them all is what overflowed - STRING_AGG on wide tables (issue #735). Skip them: report no columns - for a CCI. The nvarchar(max) cast still guards wide *nonclustered* - columnstore indexes (type 6), whose columns ARE user-chosen identity. */ - select string_agg(cast(col.[name] as nvarchar(max)), ', ') within group (order by ic.key_ordinal) as cols - from sys.index_columns ic {{ information_schema_hints() }} - inner join sys.columns col {{ information_schema_hints() }} - on col.object_id = ic.object_id and col.column_id = ic.column_id - where ic.object_id = i.object_id and ic.index_id = i.index_id - and ic.is_included_column = 0 - and i.[type] <> 5 - ) key_cols - outer apply ( - select string_agg(cast(col.[name] as nvarchar(max)), ', ') as cols - from sys.index_columns ic {{ information_schema_hints() }} - inner join sys.columns col {{ information_schema_hints() }} - on col.object_id = ic.object_id and col.column_id = ic.column_id - where ic.object_id = i.object_id and ic.index_id = i.index_id - and ic.is_included_column = 1 - ) incl_cols - outer apply ( - select string_agg(cast(col.[name] as nvarchar(max)), ', ') as cols - from sys.index_columns ic {{ information_schema_hints() }} - inner join sys.columns col {{ information_schema_hints() }} - on col.object_id = ic.object_id and col.column_id = ic.column_id - where ic.object_id = i.object_id and ic.index_id = i.index_id - and ic.is_descending_key = 1 - ) desc_cols - outer apply ( - /* MAX() rather than TOP 1: deterministic if partitions ever carry - mixed compression (the adapter doesn't manage partitioning today) */ - select max(p.data_compression_desc) as data_compression_desc - from sys.partitions p {{ information_schema_hints() }} - where p.object_id = i.object_id and p.index_id = i.index_id - ) part - where i.object_id = OBJECT_ID('{{ relation.schema }}.{{ relation.identifier }}') - and i.index_id > 0 - and i.[type] not in (3, 4, 7) /* xml, spatial, memory-optimized hash */ - {%- endcall %} - {{ return(load_result('describe_indexes').table) }} -{% endmacro %} - - -{% macro sqlserver__get_drop_index_sql(relation, index_name) -%} - drop index [{{ index_name }}] on {{ relation }} -{%- endmacro %} - - -{% macro sqlserver__create_indexes(relation) %} - {#- - Override of the dbt-adapters default to validate the index set as a whole - (at most one clustered; clustered rowstore vs as_columnstore conflict) - before creating anything. as_columnstore is only honored by - create_table_as, so it is irrelevant for seeds despite defaulting true. - -#} - {%- set raw_indexes = config.get('indexes', default=[]) -%} - {%- set materialized = config.get('materialized') -%} - {%- set as_columnstore = config.get('as_columnstore', default=true) - if materialized in ('table', 'incremental', 'snapshot') else false -%} - {%- do adapter.validate_indexes( - raw_indexes, as_columnstore, config.get('drop_unmanaged_indexes', default=false) - ) -%} - {%- for _index_dict in raw_indexes %} - {%- set create_index_sql = get_create_index_sql(relation, _index_dict) -%} - {% if create_index_sql %} - {% do run_query(create_index_sql) %} - {% endif %} - {%- endfor %} -{% endmacro %} - - -{% macro sqlserver__reconcile_indexes(relation) %} - {#- - Converge an existing relation on its configured index set. Called on the - paths where the relation persists across runs (incremental non-full- - refresh, dml table refresh, snapshot updates), where create_indexes alone - would let config changes drift. - -#} - {%- set raw_indexes = config.get('indexes', default=[]) -%} - {%- set drop_unmanaged = config.get('drop_unmanaged_indexes', default=false) -%} - {#- all three callers (incremental, dml refresh, snapshot) honor as_columnstore -#} - {%- do adapter.validate_indexes( - raw_indexes, config.get('as_columnstore', default=true), drop_unmanaged - ) -%} - {%- set existing = sqlserver__describe_indexes(relation) -%} - {%- set result = adapter.index_changes(existing, raw_indexes, relation, drop_unmanaged) -%} - {%- for warning in result['warnings'] %} - {% do log("Index reconcile on " ~ relation ~ ": " ~ warning, info=true) %} - {%- endfor %} - {#- Apply all drops and creates in ONE transactional batch: a definition - change is then atomic, with no window where a replacement index (or the - uniqueness it enforces) is missing for concurrent readers. xact_abort - guarantees rollback if any statement fails mid-batch. -#} - {%- set reconcile_statements = [] -%} - {%- for index_name in result['drops'] %} - {% do log("Dropping index " ~ index_name ~ " on " ~ relation, info=true) %} - {%- do reconcile_statements.append(sqlserver__get_drop_index_sql(relation, index_name)) -%} - {%- endfor %} - {%- for index_dict in result['creates'] %} - {%- do reconcile_statements.append(sqlserver__get_create_index_sql(relation, index_dict)) -%} - {%- endfor %} - {% if reconcile_statements %} - {% do run_query( - "set xact_abort on;\nbegin transaction;\n" - ~ reconcile_statements | join(";\n") - ~ ";\ncommit transaction;" - ) %} - {% endif %} - {#- ONLINE / RESUMABLE creates cannot run inside the transaction above - (SQL Server forbids RESUMABLE in a user transaction, and an ONLINE build - wrapped in one holds its locks until commit, negating the non-blocking - intent). Apply them individually in autocommit. The drops above have - already committed, so a replacement index still builds after its - predecessor is gone; these are not part of the atomic batch, so there is - a brief window where the new index is absent for readers. -#} - {%- for index_dict in result['creates_no_txn'] %} - {% do log("Creating ONLINE/RESUMABLE index outside the reconcile transaction on " ~ relation, info=true) %} - {% do run_query(sqlserver__get_create_index_sql(relation, index_dict)) %} - {%- endfor %} -{% endmacro %} +{% macro mssql__quote_ident(name) -%} + {{ return('[' ~ name | replace(']', ']]') ~ ']') }} +{%- endmacro %} + + +{% macro mssql__qualified_relation(rel) -%} + {{ return( + mssql__quote_ident(rel.database) ~ '.' ~ + mssql__quote_ident(rel.schema) ~ '.' ~ + mssql__quote_ident(rel.identifier) + ) }} +{%- endmacro %} + + +{% macro mssql__strip_dbt_suffix(identifier) -%} + {%- set ns = namespace(result=identifier) -%} + {%- for suffix in ['__dbt_tmp_vw', '__dbt_backup', '__dbt_tmp'] -%} + {%- if ns.result.endswith(suffix) -%} + {%- set ns.result = ns.result[:(ns.result | length) - (suffix | length)] -%} + {%- endif -%} + {%- endfor -%} + {{ return(ns.result) }} +{%- endmacro %} + + +{% macro mssql__index_name(rel, type, columns, unique=False, includes=false) -%} + {%- set cols = [columns] if columns is string else columns -%} + {%- set incs = [] if not includes else ([includes] if includes is string else includes) -%} + {%- set stripped = mssql__strip_dbt_suffix(rel.identifier) | replace('[', '') | replace(']', '') -%} + {%- set sig = type ~ '|' ~ (cols | join(',')) ~ '|' ~ (unique | string) ~ '|' ~ (incs | join(',')) -%} + {%- set hash = local_md5(sig) -%} + {%- set prefix = type ~ '_' ~ stripped -%} + {%- set max_prefix = 83 -%} + {%- if prefix | length > max_prefix -%} + {%- set prefix = (prefix | list)[:max_prefix] | join -%} + {%- endif -%} + {{ return(prefix ~ '_' ~ hash) }} +{%- endmacro %} + + +{% macro sqlserver__index_exists(rel, index_name) -%} + EXISTS ( + SELECT 1 + FROM sys.indexes {{ information_schema_hints() }} + WHERE name = N'{{ index_name }}' + AND object_id = OBJECT_ID(N'{{ mssql__qualified_relation(rel) }}') + ) +{%- endmacro %} + + +{% macro sqlserver__create_clustered_columnstore_index(relation) -%} + {%- set stripped_id = mssql__strip_dbt_suffix(relation.identifier) | replace('[', '') | replace(']', '') -%} + {%- set cci_name = relation.schema | replace('[', '') | replace(']', '') ~ '_' ~ stripped_id ~ '_cci' -%} + {%- set cci_literal = cci_name | replace("'", "''") -%} + {%- set full_relation_literal = mssql__qualified_relation(relation) | replace("'", "''") -%} + {{ get_use_database_sql(relation.database) }} + if EXISTS ( + SELECT * + FROM sys.indexes {{ information_schema_hints() }} + WHERE name = N'{{ cci_literal }}' + AND object_id = OBJECT_ID(N'{{ full_relation_literal }}') + ) + DROP INDEX {{ mssql__quote_ident(cci_name) }} ON {{ mssql__qualified_relation(relation) }} + CREATE CLUSTERED COLUMNSTORE INDEX {{ mssql__quote_ident(cci_name) }} + ON {{ mssql__qualified_relation(relation) }} +{% endmacro %} + +{% macro drop_xml_indexes() -%} + {{ log("Running drop_xml_indexes() macro...") }} + + declare @drop_xml_indexes nvarchar(max); + select @drop_xml_indexes = ( + select 'IF INDEXPROPERTY(' + CONVERT(VARCHAR(MAX), sys.tables.[object_id]) + ', ''' + sys.indexes.[name] + ''', ''IndexId'') IS NOT NULL DROP INDEX [' + sys.indexes.[name] + '] ON ' + '[' + SCHEMA_NAME(sys.tables.[schema_id]) + '].[' + OBJECT_NAME(sys.tables.[object_id]) + ']; ' + from sys.indexes {{ information_schema_hints() }} + inner join sys.tables {{ information_schema_hints() }} + on sys.indexes.object_id = sys.tables.object_id + where sys.indexes.[name] is not null + and sys.indexes.type_desc = 'XML' + and sys.tables.[name] = '{{ this.table }}' + for xml path('') + ); exec sp_executesql @drop_xml_indexes; +{%- endmacro %} + + +{% macro drop_spatial_indexes() -%} + {# Altered from https://stackoverflow.com/q/1344401/10415173 #} + {# and https://stackoverflow.com/a/33785833/10415173 #} + + {{ log("Running drop_spatial_indexes() macro...") }} + + declare @drop_spatial_indexes nvarchar(max); + select @drop_spatial_indexes = ( + select 'IF INDEXPROPERTY(' + CONVERT(VARCHAR(MAX), sys.tables.[object_id]) + ', ''' + sys.indexes.[name] + ''', ''IndexId'') IS NOT NULL DROP INDEX [' + sys.indexes.[name] + '] ON ' + '[' + SCHEMA_NAME(sys.tables.[schema_id]) + '].[' + OBJECT_NAME(sys.tables.[object_id]) + ']; ' + from sys.indexes {{ information_schema_hints() }} + inner join sys.tables {{ information_schema_hints() }} + on sys.indexes.object_id = sys.tables.object_id + where sys.indexes.[name] is not null + and sys.indexes.type_desc = 'Spatial' + and sys.tables.[name] = '{{ this.table }}' + for xml path('') + ); exec sp_executesql @drop_spatial_indexes; +{%- endmacro %} + + +{% macro drop_fk_constraints() -%} + {# Altered from https://stackoverflow.com/q/1344401/10415173 #} + + {{ log("Running drop_fk_constraints() macro...") }} + + declare @drop_fk_constraints nvarchar(max); + select @drop_fk_constraints = ( + select 'IF OBJECT_ID(''' + SCHEMA_NAME(CONVERT(VARCHAR(MAX), sys.foreign_keys.[schema_id])) + '.' + sys.foreign_keys.[name] + ''', ''F'') IS NOT NULL ALTER TABLE [' + SCHEMA_NAME(sys.foreign_keys.[schema_id]) + '].[' + OBJECT_NAME(sys.foreign_keys.[parent_object_id]) + '] DROP CONSTRAINT [' + sys.foreign_keys.[name]+ '];' + from sys.foreign_keys + inner join sys.tables on sys.foreign_keys.[referenced_object_id] = sys.tables.[object_id] + where sys.tables.[name] = '{{ this.table }}' + for xml path('') + ); exec sp_executesql @drop_fk_constraints; + +{%- endmacro %} + + +{% macro drop_pk_constraints() -%} + {# Altered from https://stackoverflow.com/q/1344401/10415173 #} + {# and https://stackoverflow.com/a/33785833/10415173 #} + + {{ drop_xml_indexes() }} + + {{ drop_spatial_indexes() }} + + {{ drop_fk_constraints() }} + + {{ log("Running drop_pk_constraints() macro...") }} + + declare @drop_pk_constraints nvarchar(max); + select @drop_pk_constraints = ( + select 'IF INDEXPROPERTY(' + CONVERT(VARCHAR(MAX), sys.tables.[object_id]) + ', ''' + sys.indexes.[name] + ''', ''IndexId'') IS NOT NULL ALTER TABLE [' + SCHEMA_NAME(sys.tables.[schema_id]) + '].[' + sys.tables.[name] + '] DROP CONSTRAINT [' + sys.indexes.[name]+ '];' + from sys.indexes + inner join sys.tables on sys.indexes.[object_id] = sys.tables.[object_id] + where sys.indexes.is_primary_key = 1 + and sys.tables.[name] = '{{ this.table }}' + for xml path('') + ); exec sp_executesql @drop_pk_constraints; + +{%- endmacro %} + + +{% macro drop_all_indexes_on_table() -%} + {# Altered from https://stackoverflow.com/q/1344401/10415173 #} + {# and https://stackoverflow.com/a/33785833/10415173 #} + + {{ drop_pk_constraints() }} + + {{ log("Dropping remaining indexes...") }} + + declare @drop_remaining_indexes_last nvarchar(max); + select @drop_remaining_indexes_last = ( + select 'IF INDEXPROPERTY(' + CONVERT(VARCHAR(MAX), sys.tables.[object_id]) + ', ''' + sys.indexes.[name] + ''', ''IndexId'') IS NOT NULL DROP INDEX [' + sys.indexes.[name] + '] ON ' + '[' + SCHEMA_NAME(sys.tables.[schema_id]) + '].[' + OBJECT_NAME(sys.tables.[object_id]) + ']; ' + from sys.indexes {{ information_schema_hints() }} + inner join sys.tables {{ information_schema_hints() }} + on sys.indexes.object_id = sys.tables.object_id + where sys.indexes.[name] is not null + and SCHEMA_NAME(sys.tables.schema_id) = '{{ this.schema }}' + and sys.tables.[name] = '{{ this.table }}' + for xml path('') + ); exec sp_executesql @drop_remaining_indexes_last; + +{%- endmacro %} + + +{% macro create_clustered_index(columns, unique=False, relation=none) -%} + {%- set _relation = relation if relation is not none else this -%} + {%- set cols = [columns] if columns is string else columns -%} + {%- set idx_name = mssql__index_name(_relation, 'cidx', cols, unique=unique) -%} + {%- set quoted_cols = [] -%} + {%- for col in cols -%} + {%- do quoted_cols.append(mssql__quote_ident(col)) -%} + {%- endfor -%} + {{ log("Creating clustered index...") }} + + if not exists(select 1 + from sys.indexes {{ information_schema_hints() }} + where name = N'{{ idx_name }}' + and object_id = OBJECT_ID(N'{{ mssql__qualified_relation(_relation) }}') + ) + begin + + create + {% if unique -%} + unique + {% endif %} + clustered index + [{{ idx_name }}] + on {{ mssql__qualified_relation(_relation) }} ({{ quoted_cols | join(', ') }}) + end +{%- endmacro %} + + +{% macro create_nonclustered_index(columns, includes=False, relation=none) %} + + {%- set _relation = relation if relation is not none else this -%} + {%- set cols = [columns] if columns is string else columns -%} + {%- set incs = [] if not includes else ([includes] if includes is string else includes) -%} + {%- set idx_name = mssql__index_name(_relation, 'nidx', cols, includes=incs) -%} + {%- set quoted_cols = [] -%} + {%- for col in cols -%} + {%- do quoted_cols.append(mssql__quote_ident(col)) -%} + {%- endfor -%} + {%- set quoted_incs = [] -%} + {%- for inc in incs -%} + {%- do quoted_incs.append(mssql__quote_ident(inc)) -%} + {%- endfor -%} + {{ log("Creating nonclustered index...") }} + + if not exists(select 1 + from sys.indexes {{ information_schema_hints() }} + where name = N'{{ idx_name }}' + and object_id = OBJECT_ID(N'{{ mssql__qualified_relation(_relation) }}') + ) + begin + create nonclustered index + [{{ idx_name }}] + on {{ mssql__qualified_relation(_relation) }} ({{ quoted_cols | join(', ') }}) + {% if quoted_incs -%} + include ({{ quoted_incs | join(', ') }}) + {% endif %} + end +{% endmacro %} + + +{% macro drop_fk_indexes_on_table(relation) -%} + {% call statement('find_references', fetch_result=true) %} + USE [{{ relation.database }}]; + SELECT obj.name AS FK_NAME, + sch.name AS [schema_name], + tab1.name AS [table], + col1.name AS [column], + tab2.name AS [referenced_table], + col2.name AS [referenced_column] + FROM sys.foreign_key_columns fkc + INNER JOIN sys.objects obj + ON obj.object_id = fkc.constraint_object_id + INNER JOIN sys.tables tab1 + ON tab1.object_id = fkc.parent_object_id + INNER JOIN sys.schemas sch + ON tab1.schema_id = sch.schema_id + INNER JOIN sys.columns col1 + ON col1.column_id = parent_column_id AND col1.object_id = tab1.object_id + INNER JOIN sys.tables tab2 + ON tab2.object_id = fkc.referenced_object_id + INNER JOIN sys.columns col2 + ON col2.column_id = referenced_column_id AND col2.object_id = tab2.object_id + WHERE sch.name = '{{ relation.schema }}' and tab2.name = '{{ relation.identifier }}' + {% endcall %} + {% set references = load_result('find_references')['data'] %} + {% for reference in references -%} + {% call statement('main') -%} + alter table [{{reference[1]}}].[{{reference[2]}}] drop constraint [{{reference[0]}}] + {%- endcall %} + {% endfor %} +{% endmacro %} + +{% macro sqlserver__list_nonclustered_rowstore_indexes(relation) -%} + {% call statement('list_nonclustered_rowstore_indexes', fetch_result=True) -%} + + SELECT i.name AS index_name + , i.name + '__dbt_backup' as index_new_name + , COL_NAME(ic.object_id,ic.column_id) AS column_name + FROM sys.indexes AS i + INNER JOIN sys.index_columns AS ic + ON i.object_id = ic.object_id AND i.index_id = ic.index_id and i.type <> 5 + WHERE i.object_id = OBJECT_ID('{{ relation.schema }}.{{ relation.identifier }}') + + UNION ALL + + SELECT obj.name AS index_name + , obj.name + '__dbt_backup' as index_new_name + , col1.name AS column_name + FROM sys.foreign_key_columns fkc + INNER JOIN sys.objects obj + ON obj.object_id = fkc.constraint_object_id + INNER JOIN sys.tables tab1 + ON tab1.object_id = fkc.parent_object_id + INNER JOIN sys.schemas sch + ON tab1.schema_id = sch.schema_id + INNER JOIN sys.columns col1 + ON col1.column_id = parent_column_id AND col1.object_id = tab1.object_id + INNER JOIN sys.tables tab2 + ON tab2.object_id = fkc.referenced_object_id + INNER JOIN sys.columns col2 + ON col2.column_id = referenced_column_id AND col2.object_id = tab2.object_id + WHERE sch.name = '{{ relation.schema }}' and tab1.name = '{{ relation.identifier }}' + + {% endcall %} + {{ return(load_result('list_nonclustered_rowstore_indexes').table) }} +{% endmacro %} + + +{% macro sqlserver__server_major_version() -%} + {#- Detected engine major version: 13 = 2016, 14 = 2017, 15 = 2019, + 16 = 2022, 17 = 2025. parsename() on the always-4-part productversion + works on every supported release, unlike + SERVERPROPERTY('ProductMajorVersion') which is 2014+ only. Returns none + outside an executing context (e.g. parse time). -#} + {%- if not execute -%}{{ return(none) }}{%- endif -%} + {%- set result = run_query( + "select cast(parsename(cast(serverproperty('productversion') as varchar(128)), 4) as int) as major_version" + ) -%} + {{ return(result.columns[0].values()[0]) }} +{%- endmacro %} + + +{% macro sqlserver__get_create_index_sql(relation, index_dict) -%} + {%- set index_config = adapter.parse_index(index_dict) -%} + {%- set index_name = index_config.render(relation) -%} + + {# Validations are made on the adapter class SQLServerIndexConfig to control resulting sql #} + {# Names are a deterministic hash of the full definition, so an existing #} + {# index with this name is already the index we want: skip, don't fail. #} + if not exists(select * + from sys.indexes {{ information_schema_hints() }} + where name = '{{ index_name }}' + and object_id = OBJECT_ID('{{ relation }}') + ) + begin + {# key columns: bracket-quoted (with ]] escaping) plus per-column direction #} + {%- set key_columns = [] -%} + {%- for column in index_config.columns -%} + {%- do key_columns.append( + '[' ~ column | replace(']', ']]') ~ ']' + ~ (' desc' if column in index_config.descending_columns else '') + ) -%} + {%- endfor -%} + {%- set include_columns = [] -%} + {%- for column in index_config.included_columns -%} + {%- do include_columns.append('[' ~ column | replace(']', ']]') ~ ']') -%} + {%- endfor %} + create + {% if index_config.unique -%} unique {% endif %}{{ index_config.type }} + index [{{ index_name }}] + on {{ relation }} + ({{ key_columns | join(', ') }}) + {% if include_columns -%} + include ({{ include_columns | join(', ') }}) + {% endif %} + {% if index_config.where -%} + where {{ index_config.where }} + {% endif %} + {#- optimize_for_sequential_key, RESUMABLE and RESUMABLE's MAX_DURATION are + CREATE INDEX options that SQL Server only recognizes on 2019 (major 15) + and newer. This adapter still supports 2017/2016, so on older engines + they are dropped (with a warning) and the index is built without them, + rather than failing with "is not a recognized CREATE INDEX option". + ONLINE is intentionally kept: it is recognized on 2017 (edition-gated, + not version-gated). The server version is only queried when one of these + options is actually requested, so the common path adds no round-trip. -#} + {%- set v2019_only_build_options = ['resumable', 'max_duration'] -%} + {%- set _build_options = index_config.build_options or {} -%} + {%- set _wants_v2019_option = index_config.optimize_for_sequential_key + or _build_options.get('resumable') or _build_options.get('max_duration') -%} + {%- set drop_v2019_options = false -%} + {%- if _wants_v2019_option -%} + {%- set _major = sqlserver__server_major_version() -%} + {%- if _major is not none and _major < 15 -%} + {%- set drop_v2019_options = true -%} + {%- do log( + "Index [" ~ index_name ~ "] on " ~ relation ~ ": optimize_for_sequential_key" + ~ " / resumable require SQL Server 2019 (15.x) or newer; building the index" + ~ " without them on detected major version " ~ _major ~ ".", info=true + ) -%} + {%- endif -%} + {%- endif -%} + {%- set with_options = [] -%} + {%- if index_config.data_compression -%} + {%- do with_options.append('data_compression = ' ~ index_config.data_compression | upper) -%} + {%- endif -%} + {%- if index_config.fillfactor -%} + {%- do with_options.append('fillfactor = ' ~ index_config.fillfactor) -%} + {%- endif -%} + {%- if index_config.pad_index -%} + {%- do with_options.append('pad_index = on') -%} + {%- endif -%} + {%- if index_config.ignore_dup_key -%} + {%- do with_options.append('ignore_dup_key = on') -%} + {%- endif -%} + {%- if index_config.optimize_for_sequential_key and not drop_v2019_options -%} + {%- do with_options.append('optimize_for_sequential_key = on') -%} + {%- endif -%} + {%- if index_config.sort_in_tempdb -%} + {%- do with_options.append('sort_in_tempdb = on') -%} + {%- endif -%} + {%- for option_key, option_value in _build_options.items() -%} + {%- if drop_v2019_options and option_key in v2019_only_build_options -%} + {#- skipped: not recognized before SQL Server 2019 -#} + {%- elif option_value is sameas true -%} + {%- do with_options.append(option_key ~ ' = on') -%} + {%- elif option_value is sameas false -%} + {%- do with_options.append(option_key ~ ' = off') -%} + {%- elif option_key == 'max_duration' -%} + {%- do with_options.append('max_duration = ' ~ option_value ~ ' minutes') -%} + {%- else -%} + {%- do with_options.append(option_key ~ ' = ' ~ option_value) -%} + {%- endif -%} + {%- endfor -%} + {% if with_options %} + with ({{ with_options | join(', ') }}) + {% endif %} + end +{%- endmacro %} + + +{% macro sqlserver__describe_indexes(relation) %} + {% call statement('describe_indexes', fetch_result=True) -%} + select + i.[name] as [name], + case when i.[type] = 1 then 'clustered' + when i.[type] = 2 then 'nonclustered' + when i.[type] = 5 then 'clustered columnstore' + when i.[type] = 6 then 'columnstore' + end as [type], + i.is_unique as [unique], + i.is_primary_key as is_primary_key, + i.is_unique_constraint as is_unique_constraint, + isnull(key_cols.cols, '') as [columns], + isnull(incl_cols.cols, '') as included_columns, + case when i.[type] in (1, 2, 6) + then isnull(part.data_compression_desc, 'NONE') + end as data_compression, + isnull(desc_cols.cols, '') as descending_columns, + i.filter_definition as [where], + i.fill_factor as [fillfactor], + i.ignore_dup_key as [ignore_dup_key] + /* optimize_for_sequential_key is deliberately not selected: the + sys.indexes column only exists on SQL Server 2019+ and managed + comparisons are name-based, so it isn't needed here */ + from sys.indexes i {{ information_schema_hints() }} + outer apply ( + /* STRING_AGG ... WITHIN GROUP requires SQL Server 2017+, the floor + of this adapter's CI matrix */ + select string_agg(cast(col.[name] as nvarchar(max)), ', ') within group (order by ic.key_ordinal) as cols + from sys.index_columns ic {{ information_schema_hints() }} + inner join sys.columns col {{ information_schema_hints() }} + on col.object_id = ic.object_id and col.column_id = ic.column_id + where ic.object_id = i.object_id and ic.index_id = i.index_id + and ic.is_included_column = 0 + ) key_cols + outer apply ( + select string_agg(cast(col.[name] as nvarchar(max)), ', ') as cols + from sys.index_columns ic {{ information_schema_hints() }} + inner join sys.columns col {{ information_schema_hints() }} + on col.object_id = ic.object_id and col.column_id = ic.column_id + where ic.object_id = i.object_id and ic.index_id = i.index_id + and ic.is_included_column = 1 + ) incl_cols + outer apply ( + select string_agg(cast(col.[name] as nvarchar(max)), ', ') as cols + from sys.index_columns ic {{ information_schema_hints() }} + inner join sys.columns col {{ information_schema_hints() }} + on col.object_id = ic.object_id and col.column_id = ic.column_id + where ic.object_id = i.object_id and ic.index_id = i.index_id + and ic.is_descending_key = 1 + ) desc_cols + outer apply ( + /* MAX() rather than TOP 1: deterministic if partitions ever carry + mixed compression (the adapter doesn't manage partitioning today) */ + select max(p.data_compression_desc) as data_compression_desc + from sys.partitions p {{ information_schema_hints() }} + where p.object_id = i.object_id and p.index_id = i.index_id + ) part + where i.object_id = OBJECT_ID('{{ relation.schema }}.{{ relation.identifier }}') + and i.index_id > 0 + and i.[type] not in (3, 4, 7) /* xml, spatial, memory-optimized hash */ + {%- endcall %} + {{ return(load_result('describe_indexes').table) }} +{% endmacro %} + + +{% macro sqlserver__get_drop_index_sql(relation, index_name) -%} + drop index [{{ index_name }}] on {{ relation }} +{%- endmacro %} + + +{% macro sqlserver__create_indexes(relation) %} + {#- + Override of the dbt-adapters default to validate the index set as a whole + (at most one clustered; clustered rowstore vs as_columnstore conflict) + before creating anything. as_columnstore is only honored by + create_table_as, so it is irrelevant for seeds despite defaulting true. + -#} + {%- set raw_indexes = config.get('indexes', default=[]) -%} + {%- set materialized = config.get('materialized') -%} + {%- set as_columnstore = config.get('as_columnstore', default=true) + if materialized in ('table', 'incremental', 'snapshot') else false -%} + {%- do adapter.validate_indexes( + raw_indexes, as_columnstore, config.get('drop_unmanaged_indexes', default=false) + ) -%} + {%- for _index_dict in raw_indexes %} + {%- set create_index_sql = get_create_index_sql(relation, _index_dict) -%} + {% if create_index_sql %} + {% do run_query(create_index_sql) %} + {% endif %} + {%- endfor %} +{% endmacro %} + + +{% macro sqlserver__reconcile_indexes(relation) %} + {#- + Converge an existing relation on its configured index set. Called on the + paths where the relation persists across runs (incremental non-full- + refresh, dml table refresh, snapshot updates), where create_indexes alone + would let config changes drift. + -#} + {%- set raw_indexes = config.get('indexes', default=[]) -%} + {%- set drop_unmanaged = config.get('drop_unmanaged_indexes', default=false) -%} + {#- all three callers (incremental, dml refresh, snapshot) honor as_columnstore -#} + {%- do adapter.validate_indexes( + raw_indexes, config.get('as_columnstore', default=true), drop_unmanaged + ) -%} + {%- set existing = sqlserver__describe_indexes(relation) -%} + {%- set result = adapter.index_changes(existing, raw_indexes, relation, drop_unmanaged) -%} + {%- for warning in result['warnings'] %} + {% do log("Index reconcile on " ~ relation ~ ": " ~ warning, info=true) %} + {%- endfor %} + {#- Apply all drops and creates in ONE transactional batch: a definition + change is then atomic, with no window where a replacement index (or the + uniqueness it enforces) is missing for concurrent readers. xact_abort + guarantees rollback if any statement fails mid-batch. -#} + {%- set reconcile_statements = [] -%} + {%- for index_name in result['drops'] %} + {% do log("Dropping index " ~ index_name ~ " on " ~ relation, info=true) %} + {%- do reconcile_statements.append(sqlserver__get_drop_index_sql(relation, index_name)) -%} + {%- endfor %} + {%- for index_dict in result['creates'] %} + {%- do reconcile_statements.append(sqlserver__get_create_index_sql(relation, index_dict)) -%} + {%- endfor %} + {% if reconcile_statements %} + {% do run_query( + "set xact_abort on;\nbegin transaction;\n" + ~ reconcile_statements | join(";\n") + ~ ";\ncommit transaction;" + ) %} + {% endif %} + {#- ONLINE / RESUMABLE creates cannot run inside the transaction above + (SQL Server forbids RESUMABLE in a user transaction, and an ONLINE build + wrapped in one holds its locks until commit, negating the non-blocking + intent). Apply them individually in autocommit. The drops above have + already committed, so a replacement index still builds after its + predecessor is gone; these are not part of the atomic batch, so there is + a brief window where the new index is absent for readers. -#} + {%- for index_dict in result['creates_no_txn'] %} + {% do log("Creating ONLINE/RESUMABLE index outside the reconcile transaction on " ~ relation, info=true) %} + {% do run_query(sqlserver__get_create_index_sql(relation, index_dict)) %} + {%- endfor %} +{% endmacro %} diff --git a/tests/functional/adapter/mssql/test_index_macros.py b/tests/functional/adapter/mssql/test_index_macros.py index d8df11d19..15907a182 100644 --- a/tests/functional/adapter/mssql/test_index_macros.py +++ b/tests/functional/adapter/mssql/test_index_macros.py @@ -1,430 +1,430 @@ -import pytest - -from dbt.tests.util import get_connection, run_dbt -from tests.functional.adapter.mssql.test_index_config import index_count, indexes_def - -# flake8: noqa: E501 - -index_seed_csv = """id_col,data,secondary_data,tertiary_data -1,'a'",122,20 -""" - -index_schema_base_yml = """ -version: 2 -seeds: - - name: raw_data - config: - column_types: - id_col: integer - data: nvarchar(20) - secondary_data: integer - tertiary_data: bigint -""" - -model_yml = """ -version: 2 -models: - - name: index_model - - name: index_ccs_model -""" - -model_sql = """ -{{ - config({ - "materialized": 'table', - "as_columnstore": False, - "post-hook": [ - "{{ create_clustered_index(columns = ['id_col'], unique=True) }}", - "{{ create_nonclustered_index(columns = ['data']) }}", - "{{ create_nonclustered_index(columns = ['secondary_data'], includes = ['tertiary_data']) }}", - ] - }) -}} - select * from {{ ref('raw_data') }} -""" - -model_sql_ccs = """ -{{ - config({ - "materialized": 'table', - "post-hook": [ - "{{ create_nonclustered_index(columns = ['data']) }}", - "{{ create_nonclustered_index(columns = ['secondary_data'], includes = ['tertiary_data']) }}", - ] - }) -}} - select * from {{ ref('raw_data') }} -""" - -drop_schema_model = """ -{{ - config({ - "materialized": 'table', - "post-hook": [ - "{{ drop_all_indexes_on_table() }}", - ] - }) -}} -select * from {{ ref('raw_data') }} -""" - - -class TestIndex: - @pytest.fixture(scope="class") - def project_config_update(self): - return {"name": "generic_tests"} - - @pytest.fixture(scope="class") - def seeds(self): - return { - "raw_data.csv": index_seed_csv, - "schema.yml": index_schema_base_yml, - } - - @pytest.fixture(scope="class") - def models(self): - return { - "index_model.sql": model_sql, - "index_ccs_model.sql": model_sql_ccs, - "schema.yml": model_yml, - } - - def drop_artifacts(self, project): - with get_connection(project.adapter): - project.adapter.execute("DROP TABLE IF EXISTS index_model", fetch=True) - project.adapter.execute("DROP TABLE IF EXISTS index_ccs_model") - - def test_create_index(self, project): - run_dbt(["seed"]) - run_dbt(["run"]) - - with get_connection(project.adapter): - result, table = project.adapter.execute( - index_count.format(schema_name=project.created_schemas[0]), fetch=True - ) - schema_dict = {_[0]: _[1] for _ in table.rows} - expected = { - "clustered columnstore": 1, - "clustered unique": 1, - "nonclustered": 4, - } - self.drop_artifacts(project) - assert schema_dict == expected - - -class TestIndexDropsOnlySchema: - @pytest.fixture(scope="class") - def project_config_update(self): - return {"name": "generic_tests"} - - @pytest.fixture(scope="class") - def seeds(self): - return { - "raw_data.csv": index_seed_csv, - "schema.yml": index_schema_base_yml, - } - - @pytest.fixture(scope="class") - def models(self): - return { - "index_model.sql": drop_schema_model, - "index_ccs_model.sql": model_sql_ccs, - "schema.yml": model_yml, - } - - def create_table_and_index_other_schema(self, project): - _schema = project.test_schema + "other" - create_sql = f""" - USE [{project.database}]; - IF NOT EXISTS (SELECT * FROM sys.schemas WHERE name = '{_schema}') - BEGIN - EXEC('CREATE SCHEMA [{_schema}]') - END - """ - - create_table = f""" - CREATE TABLE {_schema}.index_model ( - IDCOL BIGINT - ) - """ - - create_index = f""" - CREATE INDEX sample_schema ON {_schema}.index_model (IDCOL) - """ - with get_connection(project.adapter): - project.adapter.execute(create_sql, fetch=True) - project.adapter.execute(create_table) - project.adapter.execute(create_index) - - def drop_schema_artifacts(self, project): - _schema = project.test_schema + "other" - drop_index = f"DROP INDEX IF EXISTS sample_schema ON {_schema}.index_model" - drop_table = f"DROP TABLE IF EXISTS {_schema}.index_model" - drop_schema = f"DROP SCHEMA IF EXISTS {_schema}" - - with get_connection(project.adapter): - project.adapter.execute(drop_index, fetch=True) - project.adapter.execute(drop_table) - project.adapter.execute(drop_schema) - - def validate_other_schema(self, project): - with get_connection(project.adapter): - result, table = project.adapter.execute( - indexes_def.format( - schema_name=project.test_schema + "other", table_name="index_model" - ), - fetch=True, - ) - - assert len(table.rows) == 1 - - def test_create_index(self, project): - self.create_table_and_index_other_schema(project) - run_dbt(["seed"]) - run_dbt(["run"]) - self.validate_other_schema(project) - self.drop_schema_artifacts(project) - - -# --------------------------------------------------------------------------- -# Integration tests for the refactored index subsystem. -# -# These run against a live SQL Server (credentials from test.env) and verify: -# * incremental runs produce identical, idempotent indexes -# * INCLUDE columns are registered as is_included_column = 1 -# * quoted / weird identifiers are safely bracket-escaped -# * schema-qualified models keep indexes isolated per schema -# * multi-threaded runs are race-free (deterministic names + IF NOT EXISTS) -# * CCI is created on the final relation, not the __dbt_tmp intermediate -# * CCI name on the final table never contains __dbt_tmp or __dbt_backup -# --------------------------------------------------------------------------- - - -index_columns_for_table = """ -SELECT - i.[name] AS index_name, - i.type_desc AS index_type, - c.[name] AS column_name, - ic.is_included_column AS is_included, - ic.key_ordinal AS key_ordinal -FROM sys.indexes i -INNER JOIN sys.index_columns ic - ON i.object_id = ic.object_id AND i.index_id = ic.index_id -INNER JOIN sys.columns c - ON c.object_id = ic.object_id AND c.column_id = ic.column_id -WHERE i.object_id = OBJECT_ID(N'[{database}].[{schema}].[{table}]') -ORDER BY i.[name], ic.key_ordinal, ic.is_included_column, c.[name] -""" - - -incremental_model_sql = """ -{{ - config({ - "materialized": "incremental", - "unique_key": "id_col", - "as_columnstore": False, - "post-hook": [ - "{{ create_clustered_index(columns=['id_col'], unique=True) }}", - "{{ create_nonclustered_index(columns=['data']) }}", - ], - }) -}} -select * from {{ ref('raw_data') }} -{% if is_incremental() %} -where id_col > (select coalesce(max(id_col), 0) from {{ this }}) -{% endif %} -""" - - -class TestIndexIncremental: - """Indexes should survive incremental runs (idempotent IF NOT EXISTS).""" - - @pytest.fixture(scope="class") - def seeds(self): - return {"raw_data.csv": index_seed_csv, "schema.yml": index_schema_base_yml} - - @pytest.fixture(scope="class") - def models(self): - return {"inc_model.sql": incremental_model_sql} - - def _index_rows(self, project, table_name): - sql = index_columns_for_table.format( - database=project.database, - schema=project.test_schema, - table=table_name, - ) - with get_connection(project.adapter): - _, table = project.adapter.execute(sql, fetch=True) - return list(table.rows) - - def test_indexes_stable_across_incremental_runs(self, project): - run_dbt(["seed"]) - run_dbt(["run"]) - first = self._index_rows(project, "inc_model") - - run_dbt(["run"]) - second = self._index_rows(project, "inc_model") - - assert first == second - index_names = {r[0] for r in second} - assert len(index_names) == 2 - - -include_columns_model_sql = """ -{{ - config({ - "materialized": "table", - "as_columnstore": False, - "post-hook": [ - "{{ create_nonclustered_index(columns=['secondary_data'], includes=['tertiary_data','data']) }}", - ], - }) -}} -select * from {{ ref('raw_data') }} -""" - - -class TestIndexIncludeColumns: - """INCLUDE columns must end up as is_included_column = 1 in sys.index_columns.""" - - @pytest.fixture(scope="class") - def seeds(self): - return {"raw_data.csv": index_seed_csv, "schema.yml": index_schema_base_yml} - - @pytest.fixture(scope="class") - def models(self): - return {"inc_cols_model.sql": include_columns_model_sql} - - def test_include_columns_present(self, project): - run_dbt(["seed"]) - run_dbt(["run"]) - sql = index_columns_for_table.format( - database=project.database, - schema=project.test_schema, - table="inc_cols_model", - ) - with get_connection(project.adapter): - _, table = project.adapter.execute(sql, fetch=True) - rows = list(table.rows) - keyed = [r for r in rows if r[3] == 0] - included = [r for r in rows if r[3] == 1] - assert {r[2] for r in keyed} == {"secondary_data"} - assert {r[2] for r in included} == {"tertiary_data", "data"} - - -quoted_seed_csv = """id col],weird name -1,a -2,b -""" - -quoted_schema_yml = """ -version: 2 -seeds: - - name: raw_data - config: - column_types: - "id col]": integer - "weird name": nvarchar(20) -""" - -quoted_model_sql = """ -{{ - config({ - "materialized": "table", - "as_columnstore": False, - "post-hook": [ - "{{ create_nonclustered_index(columns=['id col]', 'weird name']) }}", - ], - }) -}} -select * from {{ ref('raw_data') }} -""" - - -class TestIndexQuotedIdentifiers: - """Bracket and space characters in column names must be safely quoted.""" - - @pytest.fixture(scope="class") - def seeds(self): - return {"raw_data.csv": quoted_seed_csv, "schema.yml": quoted_schema_yml} - - @pytest.fixture(scope="class") - def models(self): - return {"quoted_model.sql": quoted_model_sql} - - def test_quoted_columns_indexed(self, project): - run_dbt(["seed"]) - run_dbt(["run"]) - sql = index_columns_for_table.format( - database=project.database, - schema=project.test_schema, - table="quoted_model", - ) - with get_connection(project.adapter): - _, table = project.adapter.execute(sql, fetch=True) - col_names = {r[2] for r in table.rows} - assert "id col]" in col_names - assert "weird name" in col_names - - -orphan_cci_check_sql = """ -SELECT COUNT(*) -FROM sys.indexes i -INNER JOIN sys.tables t ON i.object_id = t.object_id -INNER JOIN sys.schemas s ON t.schema_id = s.schema_id -WHERE s.name = '{schema}' - AND (t.name LIKE '%__dbt_tmp' OR t.name LIKE '%__dbt_backup') - AND i.type IN (5, 6) -""" - - -class TestNoOrphanColumnstoreIndex: - """Regression: CCI must be created on the *final* relation, not the - __dbt_tmp intermediate. After a successful run there should be zero CCIs - sitting on any tmp / backup relation in the test schema.""" - - @pytest.fixture(scope="class") - def seeds(self): - return {"raw_data.csv": index_seed_csv, "schema.yml": index_schema_base_yml} - - @pytest.fixture(scope="class") - def models(self): - return { - "cci_model.sql": ( - "{{ config(materialized='table', as_columnstore=True) }}\n" - "select * from {{ ref('raw_data') }}\n" - ) - } - - def test_no_orphan_cci(self, project): - run_dbt(["seed"]) - run_dbt(["run"]) - run_dbt(["run"]) - - with get_connection(project.adapter): - _, t = project.adapter.execute( - orphan_cci_check_sql.format(schema=project.test_schema), - fetch=True, - ) - assert list(t.rows)[0][0] == 0, "Orphaned CCI found on __dbt_tmp/__dbt_backup" - - cci_name_sql = """ - SELECT i.name - FROM sys.indexes i - INNER JOIN sys.tables t ON i.object_id = t.object_id - INNER JOIN sys.schemas s ON t.schema_id = s.schema_id - WHERE s.name = '{schema}' - AND t.name = 'cci_model' - AND i.type IN (5, 6) - """.format(schema=project.test_schema) - with get_connection(project.adapter): - _, t = project.adapter.execute(cci_name_sql, fetch=True) - cci_rows = list(t.rows) - assert len(cci_rows) == 1, f"Expected exactly 1 CCI on cci_model, got {cci_rows}" - cci_name = cci_rows[0][0] - assert ( - "__dbt_tmp" not in cci_name - ), f"CCI name '{cci_name}' contains __dbt_tmp - index would be orphaned after rename" - assert ( - "__dbt_backup" not in cci_name - ), f"CCI name '{cci_name}' contains __dbt_backup - index would be orphaned after rename" +import pytest + +from dbt.tests.util import get_connection, run_dbt +from tests.functional.adapter.mssql.test_index_config import index_count, indexes_def + +# flake8: noqa: E501 + +index_seed_csv = """id_col,data,secondary_data,tertiary_data +1,'a'",122,20 +""" + +index_schema_base_yml = """ +version: 2 +seeds: + - name: raw_data + config: + column_types: + id_col: integer + data: nvarchar(20) + secondary_data: integer + tertiary_data: bigint +""" + +model_yml = """ +version: 2 +models: + - name: index_model + - name: index_ccs_model +""" + +model_sql = """ +{{ + config({ + "materialized": 'table', + "as_columnstore": False, + "post-hook": [ + "{{ create_clustered_index(columns = ['id_col'], unique=True) }}", + "{{ create_nonclustered_index(columns = ['data']) }}", + "{{ create_nonclustered_index(columns = ['secondary_data'], includes = ['tertiary_data']) }}", + ] + }) +}} + select * from {{ ref('raw_data') }} +""" + +model_sql_ccs = """ +{{ + config({ + "materialized": 'table', + "post-hook": [ + "{{ create_nonclustered_index(columns = ['data']) }}", + "{{ create_nonclustered_index(columns = ['secondary_data'], includes = ['tertiary_data']) }}", + ] + }) +}} + select * from {{ ref('raw_data') }} +""" + +drop_schema_model = """ +{{ + config({ + "materialized": 'table', + "post-hook": [ + "{{ drop_all_indexes_on_table() }}", + ] + }) +}} +select * from {{ ref('raw_data') }} +""" + + +class TestIndex: + @pytest.fixture(scope="class") + def project_config_update(self): + return {"name": "generic_tests"} + + @pytest.fixture(scope="class") + def seeds(self): + return { + "raw_data.csv": index_seed_csv, + "schema.yml": index_schema_base_yml, + } + + @pytest.fixture(scope="class") + def models(self): + return { + "index_model.sql": model_sql, + "index_ccs_model.sql": model_sql_ccs, + "schema.yml": model_yml, + } + + def drop_artifacts(self, project): + with get_connection(project.adapter): + project.adapter.execute("DROP TABLE IF EXISTS index_model", fetch=True) + project.adapter.execute("DROP TABLE IF EXISTS index_ccs_model") + + def test_create_index(self, project): + run_dbt(["seed"]) + run_dbt(["run"]) + + with get_connection(project.adapter): + result, table = project.adapter.execute( + index_count.format(schema_name=project.created_schemas[0]), fetch=True + ) + schema_dict = {_[0]: _[1] for _ in table.rows} + expected = { + "clustered columnstore": 1, + "clustered unique": 1, + "nonclustered": 4, + } + self.drop_artifacts(project) + assert schema_dict == expected + + +class TestIndexDropsOnlySchema: + @pytest.fixture(scope="class") + def project_config_update(self): + return {"name": "generic_tests"} + + @pytest.fixture(scope="class") + def seeds(self): + return { + "raw_data.csv": index_seed_csv, + "schema.yml": index_schema_base_yml, + } + + @pytest.fixture(scope="class") + def models(self): + return { + "index_model.sql": drop_schema_model, + "index_ccs_model.sql": model_sql_ccs, + "schema.yml": model_yml, + } + + def create_table_and_index_other_schema(self, project): + _schema = project.test_schema + "other" + create_sql = f""" + USE [{project.database}]; + IF NOT EXISTS (SELECT * FROM sys.schemas WHERE name = '{_schema}') + BEGIN + EXEC('CREATE SCHEMA [{_schema}]') + END + """ + + create_table = f""" + CREATE TABLE {_schema}.index_model ( + IDCOL BIGINT + ) + """ + + create_index = f""" + CREATE INDEX sample_schema ON {_schema}.index_model (IDCOL) + """ + with get_connection(project.adapter): + project.adapter.execute(create_sql, fetch=True) + project.adapter.execute(create_table) + project.adapter.execute(create_index) + + def drop_schema_artifacts(self, project): + _schema = project.test_schema + "other" + drop_index = f"DROP INDEX IF EXISTS sample_schema ON {_schema}.index_model" + drop_table = f"DROP TABLE IF EXISTS {_schema}.index_model" + drop_schema = f"DROP SCHEMA IF EXISTS {_schema}" + + with get_connection(project.adapter): + project.adapter.execute(drop_index, fetch=True) + project.adapter.execute(drop_table) + project.adapter.execute(drop_schema) + + def validate_other_schema(self, project): + with get_connection(project.adapter): + result, table = project.adapter.execute( + indexes_def.format( + schema_name=project.test_schema + "other", table_name="index_model" + ), + fetch=True, + ) + + assert len(table.rows) == 1 + + def test_create_index(self, project): + self.create_table_and_index_other_schema(project) + run_dbt(["seed"]) + run_dbt(["run"]) + self.validate_other_schema(project) + self.drop_schema_artifacts(project) + + +# --------------------------------------------------------------------------- +# Integration tests for the refactored index subsystem. +# +# These run against a live SQL Server (credentials from test.env) and verify: +# * incremental runs produce identical, idempotent indexes +# * INCLUDE columns are registered as is_included_column = 1 +# * quoted / weird identifiers are safely bracket-escaped +# * schema-qualified models keep indexes isolated per schema +# * multi-threaded runs are race-free (deterministic names + IF NOT EXISTS) +# * CCI is created on the final relation, not the __dbt_tmp intermediate +# * CCI name on the final table never contains __dbt_tmp or __dbt_backup +# --------------------------------------------------------------------------- + + +index_columns_for_table = """ +SELECT + i.[name] AS index_name, + i.type_desc AS index_type, + c.[name] AS column_name, + ic.is_included_column AS is_included, + ic.key_ordinal AS key_ordinal +FROM sys.indexes i +INNER JOIN sys.index_columns ic + ON i.object_id = ic.object_id AND i.index_id = ic.index_id +INNER JOIN sys.columns c + ON c.object_id = ic.object_id AND c.column_id = ic.column_id +WHERE i.object_id = OBJECT_ID(N'[{database}].[{schema}].[{table}]') +ORDER BY i.[name], ic.key_ordinal, ic.is_included_column, c.[name] +""" + + +incremental_model_sql = """ +{{ + config({ + "materialized": "incremental", + "unique_key": "id_col", + "as_columnstore": False, + "post-hook": [ + "{{ create_clustered_index(columns=['id_col'], unique=True) }}", + "{{ create_nonclustered_index(columns=['data']) }}", + ], + }) +}} +select * from {{ ref('raw_data') }} +{% if is_incremental() %} +where id_col > (select coalesce(max(id_col), 0) from {{ this }}) +{% endif %} +""" + + +class TestIndexIncremental: + """Indexes should survive incremental runs (idempotent IF NOT EXISTS).""" + + @pytest.fixture(scope="class") + def seeds(self): + return {"raw_data.csv": index_seed_csv, "schema.yml": index_schema_base_yml} + + @pytest.fixture(scope="class") + def models(self): + return {"inc_model.sql": incremental_model_sql} + + def _index_rows(self, project, table_name): + sql = index_columns_for_table.format( + database=project.database, + schema=project.test_schema, + table=table_name, + ) + with get_connection(project.adapter): + _, table = project.adapter.execute(sql, fetch=True) + return list(table.rows) + + def test_indexes_stable_across_incremental_runs(self, project): + run_dbt(["seed"]) + run_dbt(["run"]) + first = self._index_rows(project, "inc_model") + + run_dbt(["run"]) + second = self._index_rows(project, "inc_model") + + assert first == second + index_names = {r[0] for r in second} + assert len(index_names) == 2 + + +include_columns_model_sql = """ +{{ + config({ + "materialized": "table", + "as_columnstore": False, + "post-hook": [ + "{{ create_nonclustered_index(columns=['secondary_data'], includes=['tertiary_data','data']) }}", + ], + }) +}} +select * from {{ ref('raw_data') }} +""" + + +class TestIndexIncludeColumns: + """INCLUDE columns must end up as is_included_column = 1 in sys.index_columns.""" + + @pytest.fixture(scope="class") + def seeds(self): + return {"raw_data.csv": index_seed_csv, "schema.yml": index_schema_base_yml} + + @pytest.fixture(scope="class") + def models(self): + return {"inc_cols_model.sql": include_columns_model_sql} + + def test_include_columns_present(self, project): + run_dbt(["seed"]) + run_dbt(["run"]) + sql = index_columns_for_table.format( + database=project.database, + schema=project.test_schema, + table="inc_cols_model", + ) + with get_connection(project.adapter): + _, table = project.adapter.execute(sql, fetch=True) + rows = list(table.rows) + keyed = [r for r in rows if r[3] == 0] + included = [r for r in rows if r[3] == 1] + assert {r[2] for r in keyed} == {"secondary_data"} + assert {r[2] for r in included} == {"tertiary_data", "data"} + + +quoted_seed_csv = """id col],weird name +1,a +2,b +""" + +quoted_schema_yml = """ +version: 2 +seeds: + - name: raw_data + config: + column_types: + "id col]": integer + "weird name": nvarchar(20) +""" + +quoted_model_sql = """ +{{ + config({ + "materialized": "table", + "as_columnstore": False, + "post-hook": [ + "{{ create_nonclustered_index(columns=['id col]', 'weird name']) }}", + ], + }) +}} +select * from {{ ref('raw_data') }} +""" + + +class TestIndexQuotedIdentifiers: + """Bracket and space characters in column names must be safely quoted.""" + + @pytest.fixture(scope="class") + def seeds(self): + return {"raw_data.csv": quoted_seed_csv, "schema.yml": quoted_schema_yml} + + @pytest.fixture(scope="class") + def models(self): + return {"quoted_model.sql": quoted_model_sql} + + def test_quoted_columns_indexed(self, project): + run_dbt(["seed"]) + run_dbt(["run"]) + sql = index_columns_for_table.format( + database=project.database, + schema=project.test_schema, + table="quoted_model", + ) + with get_connection(project.adapter): + _, table = project.adapter.execute(sql, fetch=True) + col_names = {r[2] for r in table.rows} + assert "id col]" in col_names + assert "weird name" in col_names + + +orphan_cci_check_sql = """ +SELECT COUNT(*) +FROM sys.indexes i +INNER JOIN sys.tables t ON i.object_id = t.object_id +INNER JOIN sys.schemas s ON t.schema_id = s.schema_id +WHERE s.name = '{schema}' + AND (t.name LIKE '%__dbt_tmp' OR t.name LIKE '%__dbt_backup') + AND i.type IN (5, 6) +""" + + +class TestNoOrphanColumnstoreIndex: + """Regression: CCI must be created on the *final* relation, not the + __dbt_tmp intermediate. After a successful run there should be zero CCIs + sitting on any tmp / backup relation in the test schema.""" + + @pytest.fixture(scope="class") + def seeds(self): + return {"raw_data.csv": index_seed_csv, "schema.yml": index_schema_base_yml} + + @pytest.fixture(scope="class") + def models(self): + return { + "cci_model.sql": ( + "{{ config(materialized='table', as_columnstore=True) }}\n" + "select * from {{ ref('raw_data') }}\n" + ) + } + + def test_no_orphan_cci(self, project): + run_dbt(["seed"]) + run_dbt(["run"]) + run_dbt(["run"]) + + with get_connection(project.adapter): + _, t = project.adapter.execute( + orphan_cci_check_sql.format(schema=project.test_schema), + fetch=True, + ) + assert list(t.rows)[0][0] == 0, "Orphaned CCI found on __dbt_tmp/__dbt_backup" + + cci_name_sql = """ + SELECT i.name + FROM sys.indexes i + INNER JOIN sys.tables t ON i.object_id = t.object_id + INNER JOIN sys.schemas s ON t.schema_id = s.schema_id + WHERE s.name = '{schema}' + AND t.name = 'cci_model' + AND i.type IN (5, 6) + """.format(schema=project.test_schema) + with get_connection(project.adapter): + _, t = project.adapter.execute(cci_name_sql, fetch=True) + cci_rows = list(t.rows) + assert len(cci_rows) == 1, f"Expected exactly 1 CCI on cci_model, got {cci_rows}" + cci_name = cci_rows[0][0] + assert ( + "__dbt_tmp" not in cci_name + ), f"CCI name '{cci_name}' contains __dbt_tmp - index would be orphaned after rename" + assert ( + "__dbt_backup" not in cci_name + ), f"CCI name '{cci_name}' contains __dbt_backup - index would be orphaned after rename" diff --git a/tests/unit/adapters/mssql/test_indexes.py b/tests/unit/adapters/mssql/test_indexes.py index 9411edb90..189ef0c06 100644 --- a/tests/unit/adapters/mssql/test_indexes.py +++ b/tests/unit/adapters/mssql/test_indexes.py @@ -1,337 +1,338 @@ -""" -Unit tests for the SQL Server index macros. - -Mirrors the style of test_generate_schema_name.py: inline Jinja, no DB. - -Coverage: - * mssql__quote_ident - bracket escaping - * mssql__qualified_relation - 3-part name assembly - * mssql__strip_dbt_suffix - __dbt_tmp / __dbt_backup stripping - * mssql__index_name - determinism, 116-char cap, hashing, - immunity to __dbt_tmp suffix - * sqlserver__index_exists - emitted SQL shape (OBJECT_ID scoped) - * create_clustered_index - quoted columns, IF NOT EXISTS wrapper - * create_nonclustered_index - INCLUDE columns, quoting, idempotence -""" - -import hashlib -import re -from pathlib import Path - -import jinja2 -import pytest -from jinja2.runtime import Macro as _Jinja2Macro - - -class _MacroReturn(BaseException): - def __init__(self, value): - self.value = value - - -_orig_macro_call = _Jinja2Macro.__call__ - - -def _patched_macro_call(self, *args, **kwargs): - try: - return _orig_macro_call(self, *args, **kwargs) - except _MacroReturn as exc: - return exc.value - - -@pytest.fixture(scope="session", autouse=True) -def _patch_jinja2_macro_return(): - _Jinja2Macro.__call__ = _patched_macro_call - yield - _Jinja2Macro.__call__ = _orig_macro_call - - -MACRO_PATH = ( - Path(__file__).resolve().parents[4] - / "dbt" - / "include" - / "sqlserver" - / "macros" - / "adapters" - / "indexes.sql" -) - -MACRO_SRC = MACRO_PATH.read_text(encoding="utf-8") - - -class _FakeRelation: - """Minimal stand-in for dbt's Relation object.""" - - def __init__(self, database, schema, identifier): - self.database = database - self.schema = schema - self.identifier = identifier - - def __str__(self): - return f"[{self.database}].[{self.schema}].[{self.identifier}]" - - -def _env(): - env = jinja2.Environment( - trim_blocks=True, - lstrip_blocks=True, - extensions=["jinja2.ext.do"], - ) - - def local_md5(s): - return hashlib.md5(s.encode("utf-8")).hexdigest() - - env.globals.update( - { - "local_md5": local_md5, - "information_schema_hints": lambda: "", - "log": lambda *a, **kw: "", - "this": _FakeRelation("mydb", "myschema", "my_model"), - "return": lambda v: (_ for _ in ()).throw(_MacroReturn(v)), - } - ) - return env - - -def _render(call_expr, **ctx): - env = _env() - template = env.from_string(MACRO_SRC + "\n" + "{{ " + call_expr + " }}") - return template.render(**ctx).strip() - - -def _normalize_ws(s): - return re.sub(r"\s+", " ", s).strip() - - -class TestQuoteIdent: - def test_simple_name(self): - assert _render("mssql__quote_ident('foo')") == "[foo]" - - def test_name_with_space(self): - assert _render("mssql__quote_ident('weird name')") == "[weird name]" - - def test_name_with_close_bracket_is_escaped(self): - assert _render("mssql__quote_ident('we][ird')") == "[we]][ird]" - - def test_name_with_dot_is_not_split(self): - assert _render("mssql__quote_ident('a.b')") == "[a.b]" - - -class TestQualifiedRelation: - def test_three_part_name(self): - rel = _FakeRelation("mydb", "myschema", "my_model") - out = _render("mssql__qualified_relation(rel)", rel=rel) - assert out == "[mydb].[myschema].[my_model]" - - def test_escapes_brackets_in_every_part(self): - rel = _FakeRelation("d]b", "sch]ema", "id]ent") - out = _render("mssql__qualified_relation(rel)", rel=rel) - assert out == "[d]]b].[sch]]ema].[id]]ent]" - - -class TestStripDbtSuffix: - @pytest.mark.parametrize( - "identifier, expected", - [ - ("my_model", "my_model"), - ("my_model__dbt_tmp", "my_model"), - ("my_model__dbt_backup", "my_model"), - ("my_model__dbt_tmp_vw", "my_model"), - ("my__dbt_tmp_model", "my__dbt_tmp_model"), # __dbt_tmp in middle, not a suffix - ], - ) - def test_strip(self, identifier, expected): - assert _render(f"mssql__strip_dbt_suffix('{identifier}')") == expected - - -class TestIndexName: - def test_deterministic(self): - rel = _FakeRelation("d", "s", "t") - a = _render("mssql__index_name(rel, 'nidx', ['x','y'])", rel=rel) - b = _render("mssql__index_name(rel, 'nidx', ['x','y'])", rel=rel) - assert a == b - - def test_column_order_changes_name(self): - rel = _FakeRelation("d", "s", "t") - a = _render("mssql__index_name(rel, 'nidx', ['x','y'])", rel=rel) - b = _render("mssql__index_name(rel, 'nidx', ['y','x'])", rel=rel) - assert a != b - - def test_unique_flag_changes_name(self): - rel = _FakeRelation("d", "s", "t") - a = _render("mssql__index_name(rel, 'cidx', ['x'], unique=False)", rel=rel) - b = _render("mssql__index_name(rel, 'cidx', ['x'], unique=True)", rel=rel) - assert a != b - - def test_includes_changes_name(self): - rel = _FakeRelation("d", "s", "t") - a = _render("mssql__index_name(rel, 'nidx', ['x'])", rel=rel) - b = _render("mssql__index_name(rel, 'nidx', ['x'], includes=['y'])", rel=rel) - assert a != b - - def test_accepts_string_columns(self): - rel = _FakeRelation("d", "s", "t") - name = _render("mssql__index_name(rel, 'nidx', 'x')", rel=rel) - assert name.startswith("nidx_t_") - - def test_false_includes_are_treated_as_empty(self): - rel = _FakeRelation("d", "s", "t") - a = _render("mssql__index_name(rel, 'nidx', ['x'], includes=false)", rel=rel) - b = _render("mssql__index_name(rel, 'nidx', ['x'])", rel=rel) - assert a == b - - def test_dbt_tmp_suffix_does_not_affect_name(self): - a = _render( - "mssql__index_name(rel, 'cci', ['__all__'])", - rel=_FakeRelation("d", "s", "my_model"), - ) - b = _render( - "mssql__index_name(rel, 'cci', ['__all__'])", - rel=_FakeRelation("d", "s", "my_model__dbt_tmp"), - ) - c = _render( - "mssql__index_name(rel, 'cci', ['__all__'])", - rel=_FakeRelation("d", "s", "my_model__dbt_backup"), - ) - assert a == b == c - - def test_length_capped_at_116_chars(self): - rel = _FakeRelation("d", "s", "x" * 500) - name = _render("mssql__index_name(rel, 'nidx', ['c'])", rel=rel) - assert len(name) <= 116 - - def test_long_name_still_unique_per_signature(self): - rel = _FakeRelation("d", "s", "x" * 500) - a = _render("mssql__index_name(rel, 'nidx', ['c1'])", rel=rel) - b = _render("mssql__index_name(rel, 'nidx', ['c2'])", rel=rel) - assert a != b - assert len(a) <= 116 - assert len(b) <= 116 - - def test_brackets_stripped_from_readable_part(self): - rel = _FakeRelation("d", "s", "[weird]") - name = _render("mssql__index_name(rel, 'nidx', ['c'])", rel=rel) - assert "[" not in name and "]" not in name - - -class TestIndexExists: - def test_emits_object_id_scoped_check(self): - rel = _FakeRelation("mydb", "myschema", "my_model") - sql = _render("sqlserver__index_exists(rel, 'idx_foo')", rel=rel) - sql = _normalize_ws(sql) - assert "EXISTS" in sql - assert "sys.indexes" in sql - assert "name = N'idx_foo'" in sql - assert "OBJECT_ID(N'[mydb].[myschema].[my_model]')" in sql - - -class TestCreateClusteredIndex: - def test_quotes_columns(self): - rel = _FakeRelation("d", "s", "t") - sql = _render("create_clustered_index(['id_col', 'data'], relation=rel)", rel=rel) - sql = _normalize_ws(sql) - assert "([id_col], [data])" in sql - - def test_wrapped_in_if_not_exists(self): - rel = _FakeRelation("d", "s", "t") - sql = _normalize_ws(_render("create_clustered_index(['c'], relation=rel)", rel=rel)) - assert "if not exists" in sql - assert "begin create" in sql - assert sql.endswith("end") - - def test_unique_flag_emits_unique_keyword(self): - rel = _FakeRelation("d", "s", "t") - sql = _normalize_ws( - _render("create_clustered_index(['c'], unique=True, relation=rel)", rel=rel) - ) - assert "unique clustered index" in sql - - def test_accepts_string_column(self): - rel = _FakeRelation("d", "s", "t") - sql = _normalize_ws(_render("create_clustered_index('id_col', relation=rel)", rel=rel)) - assert "([id_col])" in sql - - -class TestCreateNonclusteredIndex: - def test_emits_nonclustered(self): - rel = _FakeRelation("d", "s", "t") - sql = _normalize_ws(_render("create_nonclustered_index(['c'], relation=rel)", rel=rel)) - assert "create nonclustered index" in sql - - def test_accepts_string_column(self): - rel = _FakeRelation("d", "s", "t") - sql = _normalize_ws(_render("create_nonclustered_index('c', relation=rel)", rel=rel)) - assert "([c])" in sql - - def test_accepts_string_include_column(self): - rel = _FakeRelation("d", "s", "t") - sql = _normalize_ws( - _render("create_nonclustered_index(['c'], includes='inc1', relation=rel)", rel=rel) - ) - assert "include ([inc1])" in sql - - def test_include_columns_quoted(self): - rel = _FakeRelation("d", "s", "t") - sql = _normalize_ws( - _render( - "create_nonclustered_index(['c'], includes=['inc1','inc2'], relation=rel)", - rel=rel, - ) - ) - assert "include ([inc1], [inc2])" in sql - - def test_no_include_block_when_no_includes(self): - rel = _FakeRelation("d", "s", "t") - sql = _normalize_ws(_render("create_nonclustered_index(['c'], relation=rel)", rel=rel)) - assert "include" not in sql - - def test_false_include_is_treated_as_empty(self): - rel = _FakeRelation("d", "s", "t") - sql = _normalize_ws( - _render("create_nonclustered_index(['c'], includes=false, relation=rel)", rel=rel) - ) - assert "include" not in sql - - def test_columns_with_brackets_are_escaped(self): - rel = _FakeRelation("d", "s", "t") - sql = _normalize_ws( - _render("create_nonclustered_index(['we]ird'], relation=rel)", rel=rel) - ) - assert "[we]]ird]" in sql - - def test_idempotent_wrapper(self): - rel = _FakeRelation("d", "s", "t") - sql = _normalize_ws(_render("create_nonclustered_index(['c'], relation=rel)", rel=rel)) - assert "if not exists" in sql and "begin" in sql and "end" in sql - - -class TestColumnstoreIndexName: - def test_intermediate_relation_uses_target_name(self): - intermediate = _FakeRelation("d", "s", "my_model__dbt_tmp") - target = _FakeRelation("d", "s", "my_model") - sql_int = _render( - "sqlserver__create_clustered_columnstore_index(intermediate)", - intermediate=intermediate, - ) - sql_final = _render( - "sqlserver__create_clustered_columnstore_index(target)", - target=target, - ) - name_re = re.compile( - r"CREATE\s+CLUSTERED\s+COLUMNSTORE\s+INDEX\s+(\[[^\]]+\])", re.IGNORECASE - ) - name_int = name_re.search(_normalize_ws(sql_int)).group(1) - name_final = name_re.search(_normalize_ws(sql_final)).group(1) - assert name_int == name_final, ( - f"intermediate CCI name {name_int} differs from final {name_final}; " - "would be orphaned after rename" - ) - - def test_uses_qualified_target_relation_for_create(self): - target = _FakeRelation("mydb", "myschema", "my_model") - sql = _normalize_ws( - _render( - "sqlserver__create_clustered_columnstore_index(target)", - target=target, - ) - ) - assert "ON [mydb].[myschema].[my_model]" in sql +""" +Unit tests for the SQL Server index macros. + +Mirrors the style of test_generate_schema_name.py: inline Jinja, no DB. + +Coverage: + * mssql__quote_ident - bracket escaping + * mssql__qualified_relation - 3-part name assembly + * mssql__strip_dbt_suffix - __dbt_tmp / __dbt_backup stripping + * mssql__index_name - determinism, 116-char cap, hashing, + immunity to __dbt_tmp suffix + * sqlserver__index_exists - emitted SQL shape (OBJECT_ID scoped) + * create_clustered_index - quoted columns, IF NOT EXISTS wrapper + * create_nonclustered_index - INCLUDE columns, quoting, idempotence +""" + +import hashlib +import re +from pathlib import Path + +import jinja2 +import pytest +from jinja2.runtime import Macro as _Jinja2Macro + + +class _MacroReturn(BaseException): + def __init__(self, value): + self.value = value + + +_orig_macro_call = _Jinja2Macro.__call__ + + +def _patched_macro_call(self, *args, **kwargs): + try: + return _orig_macro_call(self, *args, **kwargs) + except _MacroReturn as exc: + return exc.value + + +@pytest.fixture(scope="module", autouse=True) +def _patch_jinja2_macro_return(): + _Jinja2Macro.__call__ = _patched_macro_call + yield + _Jinja2Macro.__call__ = _orig_macro_call + + +MACRO_PATH = ( + Path(__file__).resolve().parents[4] + / "dbt" + / "include" + / "sqlserver" + / "macros" + / "adapters" + / "indexes.sql" +) + +MACRO_SRC = MACRO_PATH.read_text(encoding="utf-8") + + +class _FakeRelation: + """Minimal stand-in for dbt's Relation object.""" + + def __init__(self, database, schema, identifier): + self.database = database + self.schema = schema + self.identifier = identifier + + def __str__(self): + return f"[{self.database}].[{self.schema}].[{self.identifier}]" + + +def _env(): + env = jinja2.Environment( + trim_blocks=True, + lstrip_blocks=True, + extensions=["jinja2.ext.do"], + ) + + def local_md5(s): + return hashlib.md5(s.encode("utf-8")).hexdigest() + + env.globals.update( + { + "local_md5": local_md5, + "information_schema_hints": lambda: "", + "log": lambda *a, **kw: "", + "get_use_database_sql": lambda db: f"USE [{db}];", + "this": _FakeRelation("mydb", "myschema", "my_model"), + "return": lambda v: (_ for _ in ()).throw(_MacroReturn(v)), + } + ) + return env + + +def _render(call_expr, **ctx): + env = _env() + template = env.from_string(MACRO_SRC + "\n" + "{{ " + call_expr + " }}") + return template.render(**ctx).strip() + + +def _normalize_ws(s): + return re.sub(r"\s+", " ", s).strip() + + +class TestQuoteIdent: + def test_simple_name(self): + assert _render("mssql__quote_ident('foo')") == "[foo]" + + def test_name_with_space(self): + assert _render("mssql__quote_ident('weird name')") == "[weird name]" + + def test_name_with_close_bracket_is_escaped(self): + assert _render("mssql__quote_ident('we][ird')") == "[we]][ird]" + + def test_name_with_dot_is_not_split(self): + assert _render("mssql__quote_ident('a.b')") == "[a.b]" + + +class TestQualifiedRelation: + def test_three_part_name(self): + rel = _FakeRelation("mydb", "myschema", "my_model") + out = _render("mssql__qualified_relation(rel)", rel=rel) + assert out == "[mydb].[myschema].[my_model]" + + def test_escapes_brackets_in_every_part(self): + rel = _FakeRelation("d]b", "sch]ema", "id]ent") + out = _render("mssql__qualified_relation(rel)", rel=rel) + assert out == "[d]]b].[sch]]ema].[id]]ent]" + + +class TestStripDbtSuffix: + @pytest.mark.parametrize( + "identifier, expected", + [ + ("my_model", "my_model"), + ("my_model__dbt_tmp", "my_model"), + ("my_model__dbt_backup", "my_model"), + ("my_model__dbt_tmp_vw", "my_model"), + ("my__dbt_tmp_model", "my__dbt_tmp_model"), # __dbt_tmp in middle, not a suffix + ], + ) + def test_strip(self, identifier, expected): + assert _render(f"mssql__strip_dbt_suffix('{identifier}')") == expected + + +class TestIndexName: + def test_deterministic(self): + rel = _FakeRelation("d", "s", "t") + a = _render("mssql__index_name(rel, 'nidx', ['x','y'])", rel=rel) + b = _render("mssql__index_name(rel, 'nidx', ['x','y'])", rel=rel) + assert a == b + + def test_column_order_changes_name(self): + rel = _FakeRelation("d", "s", "t") + a = _render("mssql__index_name(rel, 'nidx', ['x','y'])", rel=rel) + b = _render("mssql__index_name(rel, 'nidx', ['y','x'])", rel=rel) + assert a != b + + def test_unique_flag_changes_name(self): + rel = _FakeRelation("d", "s", "t") + a = _render("mssql__index_name(rel, 'cidx', ['x'], unique=False)", rel=rel) + b = _render("mssql__index_name(rel, 'cidx', ['x'], unique=True)", rel=rel) + assert a != b + + def test_includes_changes_name(self): + rel = _FakeRelation("d", "s", "t") + a = _render("mssql__index_name(rel, 'nidx', ['x'])", rel=rel) + b = _render("mssql__index_name(rel, 'nidx', ['x'], includes=['y'])", rel=rel) + assert a != b + + def test_accepts_string_columns(self): + rel = _FakeRelation("d", "s", "t") + name = _render("mssql__index_name(rel, 'nidx', 'x')", rel=rel) + assert name.startswith("nidx_t_") + + def test_false_includes_are_treated_as_empty(self): + rel = _FakeRelation("d", "s", "t") + a = _render("mssql__index_name(rel, 'nidx', ['x'], includes=false)", rel=rel) + b = _render("mssql__index_name(rel, 'nidx', ['x'])", rel=rel) + assert a == b + + def test_dbt_tmp_suffix_does_not_affect_name(self): + a = _render( + "mssql__index_name(rel, 'cci', ['__all__'])", + rel=_FakeRelation("d", "s", "my_model"), + ) + b = _render( + "mssql__index_name(rel, 'cci', ['__all__'])", + rel=_FakeRelation("d", "s", "my_model__dbt_tmp"), + ) + c = _render( + "mssql__index_name(rel, 'cci', ['__all__'])", + rel=_FakeRelation("d", "s", "my_model__dbt_backup"), + ) + assert a == b == c + + def test_length_capped_at_116_chars(self): + rel = _FakeRelation("d", "s", "x" * 500) + name = _render("mssql__index_name(rel, 'nidx', ['c'])", rel=rel) + assert len(name) <= 116 + + def test_long_name_still_unique_per_signature(self): + rel = _FakeRelation("d", "s", "x" * 500) + a = _render("mssql__index_name(rel, 'nidx', ['c1'])", rel=rel) + b = _render("mssql__index_name(rel, 'nidx', ['c2'])", rel=rel) + assert a != b + assert len(a) <= 116 + assert len(b) <= 116 + + def test_brackets_stripped_from_readable_part(self): + rel = _FakeRelation("d", "s", "[weird]") + name = _render("mssql__index_name(rel, 'nidx', ['c'])", rel=rel) + assert "[" not in name and "]" not in name + + +class TestIndexExists: + def test_emits_object_id_scoped_check(self): + rel = _FakeRelation("mydb", "myschema", "my_model") + sql = _render("sqlserver__index_exists(rel, 'idx_foo')", rel=rel) + sql = _normalize_ws(sql) + assert "EXISTS" in sql + assert "sys.indexes" in sql + assert "name = N'idx_foo'" in sql + assert "OBJECT_ID(N'[mydb].[myschema].[my_model]')" in sql + + +class TestCreateClusteredIndex: + def test_quotes_columns(self): + rel = _FakeRelation("d", "s", "t") + sql = _render("create_clustered_index(['id_col', 'data'], relation=rel)", rel=rel) + sql = _normalize_ws(sql) + assert "([id_col], [data])" in sql + + def test_wrapped_in_if_not_exists(self): + rel = _FakeRelation("d", "s", "t") + sql = _normalize_ws(_render("create_clustered_index(['c'], relation=rel)", rel=rel)) + assert "if not exists" in sql + assert "begin create" in sql + assert sql.endswith("end") + + def test_unique_flag_emits_unique_keyword(self): + rel = _FakeRelation("d", "s", "t") + sql = _normalize_ws( + _render("create_clustered_index(['c'], unique=True, relation=rel)", rel=rel) + ) + assert "unique clustered index" in sql + + def test_accepts_string_column(self): + rel = _FakeRelation("d", "s", "t") + sql = _normalize_ws(_render("create_clustered_index('id_col', relation=rel)", rel=rel)) + assert "([id_col])" in sql + + +class TestCreateNonclusteredIndex: + def test_emits_nonclustered(self): + rel = _FakeRelation("d", "s", "t") + sql = _normalize_ws(_render("create_nonclustered_index(['c'], relation=rel)", rel=rel)) + assert "create nonclustered index" in sql + + def test_accepts_string_column(self): + rel = _FakeRelation("d", "s", "t") + sql = _normalize_ws(_render("create_nonclustered_index('c', relation=rel)", rel=rel)) + assert "([c])" in sql + + def test_accepts_string_include_column(self): + rel = _FakeRelation("d", "s", "t") + sql = _normalize_ws( + _render("create_nonclustered_index(['c'], includes='inc1', relation=rel)", rel=rel) + ) + assert "include ([inc1])" in sql + + def test_include_columns_quoted(self): + rel = _FakeRelation("d", "s", "t") + sql = _normalize_ws( + _render( + "create_nonclustered_index(['c'], includes=['inc1','inc2'], relation=rel)", + rel=rel, + ) + ) + assert "include ([inc1], [inc2])" in sql + + def test_no_include_block_when_no_includes(self): + rel = _FakeRelation("d", "s", "t") + sql = _normalize_ws(_render("create_nonclustered_index(['c'], relation=rel)", rel=rel)) + assert "include" not in sql + + def test_false_include_is_treated_as_empty(self): + rel = _FakeRelation("d", "s", "t") + sql = _normalize_ws( + _render("create_nonclustered_index(['c'], includes=false, relation=rel)", rel=rel) + ) + assert "include" not in sql + + def test_columns_with_brackets_are_escaped(self): + rel = _FakeRelation("d", "s", "t") + sql = _normalize_ws( + _render("create_nonclustered_index(['we]ird'], relation=rel)", rel=rel) + ) + assert "[we]]ird]" in sql + + def test_idempotent_wrapper(self): + rel = _FakeRelation("d", "s", "t") + sql = _normalize_ws(_render("create_nonclustered_index(['c'], relation=rel)", rel=rel)) + assert "if not exists" in sql and "begin" in sql and "end" in sql + + +class TestColumnstoreIndexName: + def test_intermediate_relation_uses_target_name(self): + intermediate = _FakeRelation("d", "s", "my_model__dbt_tmp") + target = _FakeRelation("d", "s", "my_model") + sql_int = _render( + "sqlserver__create_clustered_columnstore_index(intermediate)", + intermediate=intermediate, + ) + sql_final = _render( + "sqlserver__create_clustered_columnstore_index(target)", + target=target, + ) + name_re = re.compile( + r"CREATE\s+CLUSTERED\s+COLUMNSTORE\s+INDEX\s+(\[[^\]]+\])", re.IGNORECASE + ) + name_int = name_re.search(_normalize_ws(sql_int)).group(1) + name_final = name_re.search(_normalize_ws(sql_final)).group(1) + assert name_int == name_final, ( + f"intermediate CCI name {name_int} differs from final {name_final}; " + "would be orphaned after rename" + ) + + def test_uses_qualified_target_relation_for_create(self): + target = _FakeRelation("mydb", "myschema", "my_model") + sql = _normalize_ws( + _render( + "sqlserver__create_clustered_columnstore_index(target)", + target=target, + ) + ) + assert "ON [mydb].[myschema].[my_model]" in sql From e5008e099d358040842f58309cd88b4a2de5bb82 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A5rten=20Pettersson?= Date: Mon, 20 Jul 2026 12:56:46 +0000 Subject: [PATCH 4/5] chore: normalize line endings to LF --- .../sqlserver/macros/adapters/indexes.sql | 1106 ++++++++--------- .../adapter/mssql/test_index_macros.py | 860 ++++++------- tests/unit/adapters/mssql/test_indexes.py | 676 +++++----- 3 files changed, 1321 insertions(+), 1321 deletions(-) diff --git a/dbt/include/sqlserver/macros/adapters/indexes.sql b/dbt/include/sqlserver/macros/adapters/indexes.sql index b289288dd..49805eb94 100644 --- a/dbt/include/sqlserver/macros/adapters/indexes.sql +++ b/dbt/include/sqlserver/macros/adapters/indexes.sql @@ -1,553 +1,553 @@ -{% macro mssql__quote_ident(name) -%} - {{ return('[' ~ name | replace(']', ']]') ~ ']') }} -{%- endmacro %} - - -{% macro mssql__qualified_relation(rel) -%} - {{ return( - mssql__quote_ident(rel.database) ~ '.' ~ - mssql__quote_ident(rel.schema) ~ '.' ~ - mssql__quote_ident(rel.identifier) - ) }} -{%- endmacro %} - - -{% macro mssql__strip_dbt_suffix(identifier) -%} - {%- set ns = namespace(result=identifier) -%} - {%- for suffix in ['__dbt_tmp_vw', '__dbt_backup', '__dbt_tmp'] -%} - {%- if ns.result.endswith(suffix) -%} - {%- set ns.result = ns.result[:(ns.result | length) - (suffix | length)] -%} - {%- endif -%} - {%- endfor -%} - {{ return(ns.result) }} -{%- endmacro %} - - -{% macro mssql__index_name(rel, type, columns, unique=False, includes=false) -%} - {%- set cols = [columns] if columns is string else columns -%} - {%- set incs = [] if not includes else ([includes] if includes is string else includes) -%} - {%- set stripped = mssql__strip_dbt_suffix(rel.identifier) | replace('[', '') | replace(']', '') -%} - {%- set sig = type ~ '|' ~ (cols | join(',')) ~ '|' ~ (unique | string) ~ '|' ~ (incs | join(',')) -%} - {%- set hash = local_md5(sig) -%} - {%- set prefix = type ~ '_' ~ stripped -%} - {%- set max_prefix = 83 -%} - {%- if prefix | length > max_prefix -%} - {%- set prefix = (prefix | list)[:max_prefix] | join -%} - {%- endif -%} - {{ return(prefix ~ '_' ~ hash) }} -{%- endmacro %} - - -{% macro sqlserver__index_exists(rel, index_name) -%} - EXISTS ( - SELECT 1 - FROM sys.indexes {{ information_schema_hints() }} - WHERE name = N'{{ index_name }}' - AND object_id = OBJECT_ID(N'{{ mssql__qualified_relation(rel) }}') - ) -{%- endmacro %} - - -{% macro sqlserver__create_clustered_columnstore_index(relation) -%} - {%- set stripped_id = mssql__strip_dbt_suffix(relation.identifier) | replace('[', '') | replace(']', '') -%} - {%- set cci_name = relation.schema | replace('[', '') | replace(']', '') ~ '_' ~ stripped_id ~ '_cci' -%} - {%- set cci_literal = cci_name | replace("'", "''") -%} - {%- set full_relation_literal = mssql__qualified_relation(relation) | replace("'", "''") -%} - {{ get_use_database_sql(relation.database) }} - if EXISTS ( - SELECT * - FROM sys.indexes {{ information_schema_hints() }} - WHERE name = N'{{ cci_literal }}' - AND object_id = OBJECT_ID(N'{{ full_relation_literal }}') - ) - DROP INDEX {{ mssql__quote_ident(cci_name) }} ON {{ mssql__qualified_relation(relation) }} - CREATE CLUSTERED COLUMNSTORE INDEX {{ mssql__quote_ident(cci_name) }} - ON {{ mssql__qualified_relation(relation) }} -{% endmacro %} - -{% macro drop_xml_indexes() -%} - {{ log("Running drop_xml_indexes() macro...") }} - - declare @drop_xml_indexes nvarchar(max); - select @drop_xml_indexes = ( - select 'IF INDEXPROPERTY(' + CONVERT(VARCHAR(MAX), sys.tables.[object_id]) + ', ''' + sys.indexes.[name] + ''', ''IndexId'') IS NOT NULL DROP INDEX [' + sys.indexes.[name] + '] ON ' + '[' + SCHEMA_NAME(sys.tables.[schema_id]) + '].[' + OBJECT_NAME(sys.tables.[object_id]) + ']; ' - from sys.indexes {{ information_schema_hints() }} - inner join sys.tables {{ information_schema_hints() }} - on sys.indexes.object_id = sys.tables.object_id - where sys.indexes.[name] is not null - and sys.indexes.type_desc = 'XML' - and sys.tables.[name] = '{{ this.table }}' - for xml path('') - ); exec sp_executesql @drop_xml_indexes; -{%- endmacro %} - - -{% macro drop_spatial_indexes() -%} - {# Altered from https://stackoverflow.com/q/1344401/10415173 #} - {# and https://stackoverflow.com/a/33785833/10415173 #} - - {{ log("Running drop_spatial_indexes() macro...") }} - - declare @drop_spatial_indexes nvarchar(max); - select @drop_spatial_indexes = ( - select 'IF INDEXPROPERTY(' + CONVERT(VARCHAR(MAX), sys.tables.[object_id]) + ', ''' + sys.indexes.[name] + ''', ''IndexId'') IS NOT NULL DROP INDEX [' + sys.indexes.[name] + '] ON ' + '[' + SCHEMA_NAME(sys.tables.[schema_id]) + '].[' + OBJECT_NAME(sys.tables.[object_id]) + ']; ' - from sys.indexes {{ information_schema_hints() }} - inner join sys.tables {{ information_schema_hints() }} - on sys.indexes.object_id = sys.tables.object_id - where sys.indexes.[name] is not null - and sys.indexes.type_desc = 'Spatial' - and sys.tables.[name] = '{{ this.table }}' - for xml path('') - ); exec sp_executesql @drop_spatial_indexes; -{%- endmacro %} - - -{% macro drop_fk_constraints() -%} - {# Altered from https://stackoverflow.com/q/1344401/10415173 #} - - {{ log("Running drop_fk_constraints() macro...") }} - - declare @drop_fk_constraints nvarchar(max); - select @drop_fk_constraints = ( - select 'IF OBJECT_ID(''' + SCHEMA_NAME(CONVERT(VARCHAR(MAX), sys.foreign_keys.[schema_id])) + '.' + sys.foreign_keys.[name] + ''', ''F'') IS NOT NULL ALTER TABLE [' + SCHEMA_NAME(sys.foreign_keys.[schema_id]) + '].[' + OBJECT_NAME(sys.foreign_keys.[parent_object_id]) + '] DROP CONSTRAINT [' + sys.foreign_keys.[name]+ '];' - from sys.foreign_keys - inner join sys.tables on sys.foreign_keys.[referenced_object_id] = sys.tables.[object_id] - where sys.tables.[name] = '{{ this.table }}' - for xml path('') - ); exec sp_executesql @drop_fk_constraints; - -{%- endmacro %} - - -{% macro drop_pk_constraints() -%} - {# Altered from https://stackoverflow.com/q/1344401/10415173 #} - {# and https://stackoverflow.com/a/33785833/10415173 #} - - {{ drop_xml_indexes() }} - - {{ drop_spatial_indexes() }} - - {{ drop_fk_constraints() }} - - {{ log("Running drop_pk_constraints() macro...") }} - - declare @drop_pk_constraints nvarchar(max); - select @drop_pk_constraints = ( - select 'IF INDEXPROPERTY(' + CONVERT(VARCHAR(MAX), sys.tables.[object_id]) + ', ''' + sys.indexes.[name] + ''', ''IndexId'') IS NOT NULL ALTER TABLE [' + SCHEMA_NAME(sys.tables.[schema_id]) + '].[' + sys.tables.[name] + '] DROP CONSTRAINT [' + sys.indexes.[name]+ '];' - from sys.indexes - inner join sys.tables on sys.indexes.[object_id] = sys.tables.[object_id] - where sys.indexes.is_primary_key = 1 - and sys.tables.[name] = '{{ this.table }}' - for xml path('') - ); exec sp_executesql @drop_pk_constraints; - -{%- endmacro %} - - -{% macro drop_all_indexes_on_table() -%} - {# Altered from https://stackoverflow.com/q/1344401/10415173 #} - {# and https://stackoverflow.com/a/33785833/10415173 #} - - {{ drop_pk_constraints() }} - - {{ log("Dropping remaining indexes...") }} - - declare @drop_remaining_indexes_last nvarchar(max); - select @drop_remaining_indexes_last = ( - select 'IF INDEXPROPERTY(' + CONVERT(VARCHAR(MAX), sys.tables.[object_id]) + ', ''' + sys.indexes.[name] + ''', ''IndexId'') IS NOT NULL DROP INDEX [' + sys.indexes.[name] + '] ON ' + '[' + SCHEMA_NAME(sys.tables.[schema_id]) + '].[' + OBJECT_NAME(sys.tables.[object_id]) + ']; ' - from sys.indexes {{ information_schema_hints() }} - inner join sys.tables {{ information_schema_hints() }} - on sys.indexes.object_id = sys.tables.object_id - where sys.indexes.[name] is not null - and SCHEMA_NAME(sys.tables.schema_id) = '{{ this.schema }}' - and sys.tables.[name] = '{{ this.table }}' - for xml path('') - ); exec sp_executesql @drop_remaining_indexes_last; - -{%- endmacro %} - - -{% macro create_clustered_index(columns, unique=False, relation=none) -%} - {%- set _relation = relation if relation is not none else this -%} - {%- set cols = [columns] if columns is string else columns -%} - {%- set idx_name = mssql__index_name(_relation, 'cidx', cols, unique=unique) -%} - {%- set quoted_cols = [] -%} - {%- for col in cols -%} - {%- do quoted_cols.append(mssql__quote_ident(col)) -%} - {%- endfor -%} - {{ log("Creating clustered index...") }} - - if not exists(select 1 - from sys.indexes {{ information_schema_hints() }} - where name = N'{{ idx_name }}' - and object_id = OBJECT_ID(N'{{ mssql__qualified_relation(_relation) }}') - ) - begin - - create - {% if unique -%} - unique - {% endif %} - clustered index - [{{ idx_name }}] - on {{ mssql__qualified_relation(_relation) }} ({{ quoted_cols | join(', ') }}) - end -{%- endmacro %} - - -{% macro create_nonclustered_index(columns, includes=False, relation=none) %} - - {%- set _relation = relation if relation is not none else this -%} - {%- set cols = [columns] if columns is string else columns -%} - {%- set incs = [] if not includes else ([includes] if includes is string else includes) -%} - {%- set idx_name = mssql__index_name(_relation, 'nidx', cols, includes=incs) -%} - {%- set quoted_cols = [] -%} - {%- for col in cols -%} - {%- do quoted_cols.append(mssql__quote_ident(col)) -%} - {%- endfor -%} - {%- set quoted_incs = [] -%} - {%- for inc in incs -%} - {%- do quoted_incs.append(mssql__quote_ident(inc)) -%} - {%- endfor -%} - {{ log("Creating nonclustered index...") }} - - if not exists(select 1 - from sys.indexes {{ information_schema_hints() }} - where name = N'{{ idx_name }}' - and object_id = OBJECT_ID(N'{{ mssql__qualified_relation(_relation) }}') - ) - begin - create nonclustered index - [{{ idx_name }}] - on {{ mssql__qualified_relation(_relation) }} ({{ quoted_cols | join(', ') }}) - {% if quoted_incs -%} - include ({{ quoted_incs | join(', ') }}) - {% endif %} - end -{% endmacro %} - - -{% macro drop_fk_indexes_on_table(relation) -%} - {% call statement('find_references', fetch_result=true) %} - USE [{{ relation.database }}]; - SELECT obj.name AS FK_NAME, - sch.name AS [schema_name], - tab1.name AS [table], - col1.name AS [column], - tab2.name AS [referenced_table], - col2.name AS [referenced_column] - FROM sys.foreign_key_columns fkc - INNER JOIN sys.objects obj - ON obj.object_id = fkc.constraint_object_id - INNER JOIN sys.tables tab1 - ON tab1.object_id = fkc.parent_object_id - INNER JOIN sys.schemas sch - ON tab1.schema_id = sch.schema_id - INNER JOIN sys.columns col1 - ON col1.column_id = parent_column_id AND col1.object_id = tab1.object_id - INNER JOIN sys.tables tab2 - ON tab2.object_id = fkc.referenced_object_id - INNER JOIN sys.columns col2 - ON col2.column_id = referenced_column_id AND col2.object_id = tab2.object_id - WHERE sch.name = '{{ relation.schema }}' and tab2.name = '{{ relation.identifier }}' - {% endcall %} - {% set references = load_result('find_references')['data'] %} - {% for reference in references -%} - {% call statement('main') -%} - alter table [{{reference[1]}}].[{{reference[2]}}] drop constraint [{{reference[0]}}] - {%- endcall %} - {% endfor %} -{% endmacro %} - -{% macro sqlserver__list_nonclustered_rowstore_indexes(relation) -%} - {% call statement('list_nonclustered_rowstore_indexes', fetch_result=True) -%} - - SELECT i.name AS index_name - , i.name + '__dbt_backup' as index_new_name - , COL_NAME(ic.object_id,ic.column_id) AS column_name - FROM sys.indexes AS i - INNER JOIN sys.index_columns AS ic - ON i.object_id = ic.object_id AND i.index_id = ic.index_id and i.type <> 5 - WHERE i.object_id = OBJECT_ID('{{ relation.schema }}.{{ relation.identifier }}') - - UNION ALL - - SELECT obj.name AS index_name - , obj.name + '__dbt_backup' as index_new_name - , col1.name AS column_name - FROM sys.foreign_key_columns fkc - INNER JOIN sys.objects obj - ON obj.object_id = fkc.constraint_object_id - INNER JOIN sys.tables tab1 - ON tab1.object_id = fkc.parent_object_id - INNER JOIN sys.schemas sch - ON tab1.schema_id = sch.schema_id - INNER JOIN sys.columns col1 - ON col1.column_id = parent_column_id AND col1.object_id = tab1.object_id - INNER JOIN sys.tables tab2 - ON tab2.object_id = fkc.referenced_object_id - INNER JOIN sys.columns col2 - ON col2.column_id = referenced_column_id AND col2.object_id = tab2.object_id - WHERE sch.name = '{{ relation.schema }}' and tab1.name = '{{ relation.identifier }}' - - {% endcall %} - {{ return(load_result('list_nonclustered_rowstore_indexes').table) }} -{% endmacro %} - - -{% macro sqlserver__server_major_version() -%} - {#- Detected engine major version: 13 = 2016, 14 = 2017, 15 = 2019, - 16 = 2022, 17 = 2025. parsename() on the always-4-part productversion - works on every supported release, unlike - SERVERPROPERTY('ProductMajorVersion') which is 2014+ only. Returns none - outside an executing context (e.g. parse time). -#} - {%- if not execute -%}{{ return(none) }}{%- endif -%} - {%- set result = run_query( - "select cast(parsename(cast(serverproperty('productversion') as varchar(128)), 4) as int) as major_version" - ) -%} - {{ return(result.columns[0].values()[0]) }} -{%- endmacro %} - - -{% macro sqlserver__get_create_index_sql(relation, index_dict) -%} - {%- set index_config = adapter.parse_index(index_dict) -%} - {%- set index_name = index_config.render(relation) -%} - - {# Validations are made on the adapter class SQLServerIndexConfig to control resulting sql #} - {# Names are a deterministic hash of the full definition, so an existing #} - {# index with this name is already the index we want: skip, don't fail. #} - if not exists(select * - from sys.indexes {{ information_schema_hints() }} - where name = '{{ index_name }}' - and object_id = OBJECT_ID('{{ relation }}') - ) - begin - {# key columns: bracket-quoted (with ]] escaping) plus per-column direction #} - {%- set key_columns = [] -%} - {%- for column in index_config.columns -%} - {%- do key_columns.append( - '[' ~ column | replace(']', ']]') ~ ']' - ~ (' desc' if column in index_config.descending_columns else '') - ) -%} - {%- endfor -%} - {%- set include_columns = [] -%} - {%- for column in index_config.included_columns -%} - {%- do include_columns.append('[' ~ column | replace(']', ']]') ~ ']') -%} - {%- endfor %} - create - {% if index_config.unique -%} unique {% endif %}{{ index_config.type }} - index [{{ index_name }}] - on {{ relation }} - ({{ key_columns | join(', ') }}) - {% if include_columns -%} - include ({{ include_columns | join(', ') }}) - {% endif %} - {% if index_config.where -%} - where {{ index_config.where }} - {% endif %} - {#- optimize_for_sequential_key, RESUMABLE and RESUMABLE's MAX_DURATION are - CREATE INDEX options that SQL Server only recognizes on 2019 (major 15) - and newer. This adapter still supports 2017/2016, so on older engines - they are dropped (with a warning) and the index is built without them, - rather than failing with "is not a recognized CREATE INDEX option". - ONLINE is intentionally kept: it is recognized on 2017 (edition-gated, - not version-gated). The server version is only queried when one of these - options is actually requested, so the common path adds no round-trip. -#} - {%- set v2019_only_build_options = ['resumable', 'max_duration'] -%} - {%- set _build_options = index_config.build_options or {} -%} - {%- set _wants_v2019_option = index_config.optimize_for_sequential_key - or _build_options.get('resumable') or _build_options.get('max_duration') -%} - {%- set drop_v2019_options = false -%} - {%- if _wants_v2019_option -%} - {%- set _major = sqlserver__server_major_version() -%} - {%- if _major is not none and _major < 15 -%} - {%- set drop_v2019_options = true -%} - {%- do log( - "Index [" ~ index_name ~ "] on " ~ relation ~ ": optimize_for_sequential_key" - ~ " / resumable require SQL Server 2019 (15.x) or newer; building the index" - ~ " without them on detected major version " ~ _major ~ ".", info=true - ) -%} - {%- endif -%} - {%- endif -%} - {%- set with_options = [] -%} - {%- if index_config.data_compression -%} - {%- do with_options.append('data_compression = ' ~ index_config.data_compression | upper) -%} - {%- endif -%} - {%- if index_config.fillfactor -%} - {%- do with_options.append('fillfactor = ' ~ index_config.fillfactor) -%} - {%- endif -%} - {%- if index_config.pad_index -%} - {%- do with_options.append('pad_index = on') -%} - {%- endif -%} - {%- if index_config.ignore_dup_key -%} - {%- do with_options.append('ignore_dup_key = on') -%} - {%- endif -%} - {%- if index_config.optimize_for_sequential_key and not drop_v2019_options -%} - {%- do with_options.append('optimize_for_sequential_key = on') -%} - {%- endif -%} - {%- if index_config.sort_in_tempdb -%} - {%- do with_options.append('sort_in_tempdb = on') -%} - {%- endif -%} - {%- for option_key, option_value in _build_options.items() -%} - {%- if drop_v2019_options and option_key in v2019_only_build_options -%} - {#- skipped: not recognized before SQL Server 2019 -#} - {%- elif option_value is sameas true -%} - {%- do with_options.append(option_key ~ ' = on') -%} - {%- elif option_value is sameas false -%} - {%- do with_options.append(option_key ~ ' = off') -%} - {%- elif option_key == 'max_duration' -%} - {%- do with_options.append('max_duration = ' ~ option_value ~ ' minutes') -%} - {%- else -%} - {%- do with_options.append(option_key ~ ' = ' ~ option_value) -%} - {%- endif -%} - {%- endfor -%} - {% if with_options %} - with ({{ with_options | join(', ') }}) - {% endif %} - end -{%- endmacro %} - - -{% macro sqlserver__describe_indexes(relation) %} - {% call statement('describe_indexes', fetch_result=True) -%} - select - i.[name] as [name], - case when i.[type] = 1 then 'clustered' - when i.[type] = 2 then 'nonclustered' - when i.[type] = 5 then 'clustered columnstore' - when i.[type] = 6 then 'columnstore' - end as [type], - i.is_unique as [unique], - i.is_primary_key as is_primary_key, - i.is_unique_constraint as is_unique_constraint, - isnull(key_cols.cols, '') as [columns], - isnull(incl_cols.cols, '') as included_columns, - case when i.[type] in (1, 2, 6) - then isnull(part.data_compression_desc, 'NONE') - end as data_compression, - isnull(desc_cols.cols, '') as descending_columns, - i.filter_definition as [where], - i.fill_factor as [fillfactor], - i.ignore_dup_key as [ignore_dup_key] - /* optimize_for_sequential_key is deliberately not selected: the - sys.indexes column only exists on SQL Server 2019+ and managed - comparisons are name-based, so it isn't needed here */ - from sys.indexes i {{ information_schema_hints() }} - outer apply ( - /* STRING_AGG ... WITHIN GROUP requires SQL Server 2017+, the floor - of this adapter's CI matrix */ - select string_agg(cast(col.[name] as nvarchar(max)), ', ') within group (order by ic.key_ordinal) as cols - from sys.index_columns ic {{ information_schema_hints() }} - inner join sys.columns col {{ information_schema_hints() }} - on col.object_id = ic.object_id and col.column_id = ic.column_id - where ic.object_id = i.object_id and ic.index_id = i.index_id - and ic.is_included_column = 0 - ) key_cols - outer apply ( - select string_agg(cast(col.[name] as nvarchar(max)), ', ') as cols - from sys.index_columns ic {{ information_schema_hints() }} - inner join sys.columns col {{ information_schema_hints() }} - on col.object_id = ic.object_id and col.column_id = ic.column_id - where ic.object_id = i.object_id and ic.index_id = i.index_id - and ic.is_included_column = 1 - ) incl_cols - outer apply ( - select string_agg(cast(col.[name] as nvarchar(max)), ', ') as cols - from sys.index_columns ic {{ information_schema_hints() }} - inner join sys.columns col {{ information_schema_hints() }} - on col.object_id = ic.object_id and col.column_id = ic.column_id - where ic.object_id = i.object_id and ic.index_id = i.index_id - and ic.is_descending_key = 1 - ) desc_cols - outer apply ( - /* MAX() rather than TOP 1: deterministic if partitions ever carry - mixed compression (the adapter doesn't manage partitioning today) */ - select max(p.data_compression_desc) as data_compression_desc - from sys.partitions p {{ information_schema_hints() }} - where p.object_id = i.object_id and p.index_id = i.index_id - ) part - where i.object_id = OBJECT_ID('{{ relation.schema }}.{{ relation.identifier }}') - and i.index_id > 0 - and i.[type] not in (3, 4, 7) /* xml, spatial, memory-optimized hash */ - {%- endcall %} - {{ return(load_result('describe_indexes').table) }} -{% endmacro %} - - -{% macro sqlserver__get_drop_index_sql(relation, index_name) -%} - drop index [{{ index_name }}] on {{ relation }} -{%- endmacro %} - - -{% macro sqlserver__create_indexes(relation) %} - {#- - Override of the dbt-adapters default to validate the index set as a whole - (at most one clustered; clustered rowstore vs as_columnstore conflict) - before creating anything. as_columnstore is only honored by - create_table_as, so it is irrelevant for seeds despite defaulting true. - -#} - {%- set raw_indexes = config.get('indexes', default=[]) -%} - {%- set materialized = config.get('materialized') -%} - {%- set as_columnstore = config.get('as_columnstore', default=true) - if materialized in ('table', 'incremental', 'snapshot') else false -%} - {%- do adapter.validate_indexes( - raw_indexes, as_columnstore, config.get('drop_unmanaged_indexes', default=false) - ) -%} - {%- for _index_dict in raw_indexes %} - {%- set create_index_sql = get_create_index_sql(relation, _index_dict) -%} - {% if create_index_sql %} - {% do run_query(create_index_sql) %} - {% endif %} - {%- endfor %} -{% endmacro %} - - -{% macro sqlserver__reconcile_indexes(relation) %} - {#- - Converge an existing relation on its configured index set. Called on the - paths where the relation persists across runs (incremental non-full- - refresh, dml table refresh, snapshot updates), where create_indexes alone - would let config changes drift. - -#} - {%- set raw_indexes = config.get('indexes', default=[]) -%} - {%- set drop_unmanaged = config.get('drop_unmanaged_indexes', default=false) -%} - {#- all three callers (incremental, dml refresh, snapshot) honor as_columnstore -#} - {%- do adapter.validate_indexes( - raw_indexes, config.get('as_columnstore', default=true), drop_unmanaged - ) -%} - {%- set existing = sqlserver__describe_indexes(relation) -%} - {%- set result = adapter.index_changes(existing, raw_indexes, relation, drop_unmanaged) -%} - {%- for warning in result['warnings'] %} - {% do log("Index reconcile on " ~ relation ~ ": " ~ warning, info=true) %} - {%- endfor %} - {#- Apply all drops and creates in ONE transactional batch: a definition - change is then atomic, with no window where a replacement index (or the - uniqueness it enforces) is missing for concurrent readers. xact_abort - guarantees rollback if any statement fails mid-batch. -#} - {%- set reconcile_statements = [] -%} - {%- for index_name in result['drops'] %} - {% do log("Dropping index " ~ index_name ~ " on " ~ relation, info=true) %} - {%- do reconcile_statements.append(sqlserver__get_drop_index_sql(relation, index_name)) -%} - {%- endfor %} - {%- for index_dict in result['creates'] %} - {%- do reconcile_statements.append(sqlserver__get_create_index_sql(relation, index_dict)) -%} - {%- endfor %} - {% if reconcile_statements %} - {% do run_query( - "set xact_abort on;\nbegin transaction;\n" - ~ reconcile_statements | join(";\n") - ~ ";\ncommit transaction;" - ) %} - {% endif %} - {#- ONLINE / RESUMABLE creates cannot run inside the transaction above - (SQL Server forbids RESUMABLE in a user transaction, and an ONLINE build - wrapped in one holds its locks until commit, negating the non-blocking - intent). Apply them individually in autocommit. The drops above have - already committed, so a replacement index still builds after its - predecessor is gone; these are not part of the atomic batch, so there is - a brief window where the new index is absent for readers. -#} - {%- for index_dict in result['creates_no_txn'] %} - {% do log("Creating ONLINE/RESUMABLE index outside the reconcile transaction on " ~ relation, info=true) %} - {% do run_query(sqlserver__get_create_index_sql(relation, index_dict)) %} - {%- endfor %} -{% endmacro %} +{% macro mssql__quote_ident(name) -%} + {{ return('[' ~ name | replace(']', ']]') ~ ']') }} +{%- endmacro %} + + +{% macro mssql__qualified_relation(rel) -%} + {{ return( + mssql__quote_ident(rel.database) ~ '.' ~ + mssql__quote_ident(rel.schema) ~ '.' ~ + mssql__quote_ident(rel.identifier) + ) }} +{%- endmacro %} + + +{% macro mssql__strip_dbt_suffix(identifier) -%} + {%- set ns = namespace(result=identifier) -%} + {%- for suffix in ['__dbt_tmp_vw', '__dbt_backup', '__dbt_tmp'] -%} + {%- if ns.result.endswith(suffix) -%} + {%- set ns.result = ns.result[:(ns.result | length) - (suffix | length)] -%} + {%- endif -%} + {%- endfor -%} + {{ return(ns.result) }} +{%- endmacro %} + + +{% macro mssql__index_name(rel, type, columns, unique=False, includes=false) -%} + {%- set cols = [columns] if columns is string else columns -%} + {%- set incs = [] if not includes else ([includes] if includes is string else includes) -%} + {%- set stripped = mssql__strip_dbt_suffix(rel.identifier) | replace('[', '') | replace(']', '') -%} + {%- set sig = type ~ '|' ~ (cols | join(',')) ~ '|' ~ (unique | string) ~ '|' ~ (incs | join(',')) -%} + {%- set hash = local_md5(sig) -%} + {%- set prefix = type ~ '_' ~ stripped -%} + {%- set max_prefix = 83 -%} + {%- if prefix | length > max_prefix -%} + {%- set prefix = (prefix | list)[:max_prefix] | join -%} + {%- endif -%} + {{ return(prefix ~ '_' ~ hash) }} +{%- endmacro %} + + +{% macro sqlserver__index_exists(rel, index_name) -%} + EXISTS ( + SELECT 1 + FROM sys.indexes {{ information_schema_hints() }} + WHERE name = N'{{ index_name }}' + AND object_id = OBJECT_ID(N'{{ mssql__qualified_relation(rel) }}') + ) +{%- endmacro %} + + +{% macro sqlserver__create_clustered_columnstore_index(relation) -%} + {%- set stripped_id = mssql__strip_dbt_suffix(relation.identifier) | replace('[', '') | replace(']', '') -%} + {%- set cci_name = relation.schema | replace('[', '') | replace(']', '') ~ '_' ~ stripped_id ~ '_cci' -%} + {%- set cci_literal = cci_name | replace("'", "''") -%} + {%- set full_relation_literal = mssql__qualified_relation(relation) | replace("'", "''") -%} + {{ get_use_database_sql(relation.database) }} + if EXISTS ( + SELECT * + FROM sys.indexes {{ information_schema_hints() }} + WHERE name = N'{{ cci_literal }}' + AND object_id = OBJECT_ID(N'{{ full_relation_literal }}') + ) + DROP INDEX {{ mssql__quote_ident(cci_name) }} ON {{ mssql__qualified_relation(relation) }} + CREATE CLUSTERED COLUMNSTORE INDEX {{ mssql__quote_ident(cci_name) }} + ON {{ mssql__qualified_relation(relation) }} +{% endmacro %} + +{% macro drop_xml_indexes() -%} + {{ log("Running drop_xml_indexes() macro...") }} + + declare @drop_xml_indexes nvarchar(max); + select @drop_xml_indexes = ( + select 'IF INDEXPROPERTY(' + CONVERT(VARCHAR(MAX), sys.tables.[object_id]) + ', ''' + sys.indexes.[name] + ''', ''IndexId'') IS NOT NULL DROP INDEX [' + sys.indexes.[name] + '] ON ' + '[' + SCHEMA_NAME(sys.tables.[schema_id]) + '].[' + OBJECT_NAME(sys.tables.[object_id]) + ']; ' + from sys.indexes {{ information_schema_hints() }} + inner join sys.tables {{ information_schema_hints() }} + on sys.indexes.object_id = sys.tables.object_id + where sys.indexes.[name] is not null + and sys.indexes.type_desc = 'XML' + and sys.tables.[name] = '{{ this.table }}' + for xml path('') + ); exec sp_executesql @drop_xml_indexes; +{%- endmacro %} + + +{% macro drop_spatial_indexes() -%} + {# Altered from https://stackoverflow.com/q/1344401/10415173 #} + {# and https://stackoverflow.com/a/33785833/10415173 #} + + {{ log("Running drop_spatial_indexes() macro...") }} + + declare @drop_spatial_indexes nvarchar(max); + select @drop_spatial_indexes = ( + select 'IF INDEXPROPERTY(' + CONVERT(VARCHAR(MAX), sys.tables.[object_id]) + ', ''' + sys.indexes.[name] + ''', ''IndexId'') IS NOT NULL DROP INDEX [' + sys.indexes.[name] + '] ON ' + '[' + SCHEMA_NAME(sys.tables.[schema_id]) + '].[' + OBJECT_NAME(sys.tables.[object_id]) + ']; ' + from sys.indexes {{ information_schema_hints() }} + inner join sys.tables {{ information_schema_hints() }} + on sys.indexes.object_id = sys.tables.object_id + where sys.indexes.[name] is not null + and sys.indexes.type_desc = 'Spatial' + and sys.tables.[name] = '{{ this.table }}' + for xml path('') + ); exec sp_executesql @drop_spatial_indexes; +{%- endmacro %} + + +{% macro drop_fk_constraints() -%} + {# Altered from https://stackoverflow.com/q/1344401/10415173 #} + + {{ log("Running drop_fk_constraints() macro...") }} + + declare @drop_fk_constraints nvarchar(max); + select @drop_fk_constraints = ( + select 'IF OBJECT_ID(''' + SCHEMA_NAME(CONVERT(VARCHAR(MAX), sys.foreign_keys.[schema_id])) + '.' + sys.foreign_keys.[name] + ''', ''F'') IS NOT NULL ALTER TABLE [' + SCHEMA_NAME(sys.foreign_keys.[schema_id]) + '].[' + OBJECT_NAME(sys.foreign_keys.[parent_object_id]) + '] DROP CONSTRAINT [' + sys.foreign_keys.[name]+ '];' + from sys.foreign_keys + inner join sys.tables on sys.foreign_keys.[referenced_object_id] = sys.tables.[object_id] + where sys.tables.[name] = '{{ this.table }}' + for xml path('') + ); exec sp_executesql @drop_fk_constraints; + +{%- endmacro %} + + +{% macro drop_pk_constraints() -%} + {# Altered from https://stackoverflow.com/q/1344401/10415173 #} + {# and https://stackoverflow.com/a/33785833/10415173 #} + + {{ drop_xml_indexes() }} + + {{ drop_spatial_indexes() }} + + {{ drop_fk_constraints() }} + + {{ log("Running drop_pk_constraints() macro...") }} + + declare @drop_pk_constraints nvarchar(max); + select @drop_pk_constraints = ( + select 'IF INDEXPROPERTY(' + CONVERT(VARCHAR(MAX), sys.tables.[object_id]) + ', ''' + sys.indexes.[name] + ''', ''IndexId'') IS NOT NULL ALTER TABLE [' + SCHEMA_NAME(sys.tables.[schema_id]) + '].[' + sys.tables.[name] + '] DROP CONSTRAINT [' + sys.indexes.[name]+ '];' + from sys.indexes + inner join sys.tables on sys.indexes.[object_id] = sys.tables.[object_id] + where sys.indexes.is_primary_key = 1 + and sys.tables.[name] = '{{ this.table }}' + for xml path('') + ); exec sp_executesql @drop_pk_constraints; + +{%- endmacro %} + + +{% macro drop_all_indexes_on_table() -%} + {# Altered from https://stackoverflow.com/q/1344401/10415173 #} + {# and https://stackoverflow.com/a/33785833/10415173 #} + + {{ drop_pk_constraints() }} + + {{ log("Dropping remaining indexes...") }} + + declare @drop_remaining_indexes_last nvarchar(max); + select @drop_remaining_indexes_last = ( + select 'IF INDEXPROPERTY(' + CONVERT(VARCHAR(MAX), sys.tables.[object_id]) + ', ''' + sys.indexes.[name] + ''', ''IndexId'') IS NOT NULL DROP INDEX [' + sys.indexes.[name] + '] ON ' + '[' + SCHEMA_NAME(sys.tables.[schema_id]) + '].[' + OBJECT_NAME(sys.tables.[object_id]) + ']; ' + from sys.indexes {{ information_schema_hints() }} + inner join sys.tables {{ information_schema_hints() }} + on sys.indexes.object_id = sys.tables.object_id + where sys.indexes.[name] is not null + and SCHEMA_NAME(sys.tables.schema_id) = '{{ this.schema }}' + and sys.tables.[name] = '{{ this.table }}' + for xml path('') + ); exec sp_executesql @drop_remaining_indexes_last; + +{%- endmacro %} + + +{% macro create_clustered_index(columns, unique=False, relation=none) -%} + {%- set _relation = relation if relation is not none else this -%} + {%- set cols = [columns] if columns is string else columns -%} + {%- set idx_name = mssql__index_name(_relation, 'cidx', cols, unique=unique) -%} + {%- set quoted_cols = [] -%} + {%- for col in cols -%} + {%- do quoted_cols.append(mssql__quote_ident(col)) -%} + {%- endfor -%} + {{ log("Creating clustered index...") }} + + if not exists(select 1 + from sys.indexes {{ information_schema_hints() }} + where name = N'{{ idx_name }}' + and object_id = OBJECT_ID(N'{{ mssql__qualified_relation(_relation) }}') + ) + begin + + create + {% if unique -%} + unique + {% endif %} + clustered index + [{{ idx_name }}] + on {{ mssql__qualified_relation(_relation) }} ({{ quoted_cols | join(', ') }}) + end +{%- endmacro %} + + +{% macro create_nonclustered_index(columns, includes=False, relation=none) %} + + {%- set _relation = relation if relation is not none else this -%} + {%- set cols = [columns] if columns is string else columns -%} + {%- set incs = [] if not includes else ([includes] if includes is string else includes) -%} + {%- set idx_name = mssql__index_name(_relation, 'nidx', cols, includes=incs) -%} + {%- set quoted_cols = [] -%} + {%- for col in cols -%} + {%- do quoted_cols.append(mssql__quote_ident(col)) -%} + {%- endfor -%} + {%- set quoted_incs = [] -%} + {%- for inc in incs -%} + {%- do quoted_incs.append(mssql__quote_ident(inc)) -%} + {%- endfor -%} + {{ log("Creating nonclustered index...") }} + + if not exists(select 1 + from sys.indexes {{ information_schema_hints() }} + where name = N'{{ idx_name }}' + and object_id = OBJECT_ID(N'{{ mssql__qualified_relation(_relation) }}') + ) + begin + create nonclustered index + [{{ idx_name }}] + on {{ mssql__qualified_relation(_relation) }} ({{ quoted_cols | join(', ') }}) + {% if quoted_incs -%} + include ({{ quoted_incs | join(', ') }}) + {% endif %} + end +{% endmacro %} + + +{% macro drop_fk_indexes_on_table(relation) -%} + {% call statement('find_references', fetch_result=true) %} + USE [{{ relation.database }}]; + SELECT obj.name AS FK_NAME, + sch.name AS [schema_name], + tab1.name AS [table], + col1.name AS [column], + tab2.name AS [referenced_table], + col2.name AS [referenced_column] + FROM sys.foreign_key_columns fkc + INNER JOIN sys.objects obj + ON obj.object_id = fkc.constraint_object_id + INNER JOIN sys.tables tab1 + ON tab1.object_id = fkc.parent_object_id + INNER JOIN sys.schemas sch + ON tab1.schema_id = sch.schema_id + INNER JOIN sys.columns col1 + ON col1.column_id = parent_column_id AND col1.object_id = tab1.object_id + INNER JOIN sys.tables tab2 + ON tab2.object_id = fkc.referenced_object_id + INNER JOIN sys.columns col2 + ON col2.column_id = referenced_column_id AND col2.object_id = tab2.object_id + WHERE sch.name = '{{ relation.schema }}' and tab2.name = '{{ relation.identifier }}' + {% endcall %} + {% set references = load_result('find_references')['data'] %} + {% for reference in references -%} + {% call statement('main') -%} + alter table [{{reference[1]}}].[{{reference[2]}}] drop constraint [{{reference[0]}}] + {%- endcall %} + {% endfor %} +{% endmacro %} + +{% macro sqlserver__list_nonclustered_rowstore_indexes(relation) -%} + {% call statement('list_nonclustered_rowstore_indexes', fetch_result=True) -%} + + SELECT i.name AS index_name + , i.name + '__dbt_backup' as index_new_name + , COL_NAME(ic.object_id,ic.column_id) AS column_name + FROM sys.indexes AS i + INNER JOIN sys.index_columns AS ic + ON i.object_id = ic.object_id AND i.index_id = ic.index_id and i.type <> 5 + WHERE i.object_id = OBJECT_ID('{{ relation.schema }}.{{ relation.identifier }}') + + UNION ALL + + SELECT obj.name AS index_name + , obj.name + '__dbt_backup' as index_new_name + , col1.name AS column_name + FROM sys.foreign_key_columns fkc + INNER JOIN sys.objects obj + ON obj.object_id = fkc.constraint_object_id + INNER JOIN sys.tables tab1 + ON tab1.object_id = fkc.parent_object_id + INNER JOIN sys.schemas sch + ON tab1.schema_id = sch.schema_id + INNER JOIN sys.columns col1 + ON col1.column_id = parent_column_id AND col1.object_id = tab1.object_id + INNER JOIN sys.tables tab2 + ON tab2.object_id = fkc.referenced_object_id + INNER JOIN sys.columns col2 + ON col2.column_id = referenced_column_id AND col2.object_id = tab2.object_id + WHERE sch.name = '{{ relation.schema }}' and tab1.name = '{{ relation.identifier }}' + + {% endcall %} + {{ return(load_result('list_nonclustered_rowstore_indexes').table) }} +{% endmacro %} + + +{% macro sqlserver__server_major_version() -%} + {#- Detected engine major version: 13 = 2016, 14 = 2017, 15 = 2019, + 16 = 2022, 17 = 2025. parsename() on the always-4-part productversion + works on every supported release, unlike + SERVERPROPERTY('ProductMajorVersion') which is 2014+ only. Returns none + outside an executing context (e.g. parse time). -#} + {%- if not execute -%}{{ return(none) }}{%- endif -%} + {%- set result = run_query( + "select cast(parsename(cast(serverproperty('productversion') as varchar(128)), 4) as int) as major_version" + ) -%} + {{ return(result.columns[0].values()[0]) }} +{%- endmacro %} + + +{% macro sqlserver__get_create_index_sql(relation, index_dict) -%} + {%- set index_config = adapter.parse_index(index_dict) -%} + {%- set index_name = index_config.render(relation) -%} + + {# Validations are made on the adapter class SQLServerIndexConfig to control resulting sql #} + {# Names are a deterministic hash of the full definition, so an existing #} + {# index with this name is already the index we want: skip, don't fail. #} + if not exists(select * + from sys.indexes {{ information_schema_hints() }} + where name = '{{ index_name }}' + and object_id = OBJECT_ID('{{ relation }}') + ) + begin + {# key columns: bracket-quoted (with ]] escaping) plus per-column direction #} + {%- set key_columns = [] -%} + {%- for column in index_config.columns -%} + {%- do key_columns.append( + '[' ~ column | replace(']', ']]') ~ ']' + ~ (' desc' if column in index_config.descending_columns else '') + ) -%} + {%- endfor -%} + {%- set include_columns = [] -%} + {%- for column in index_config.included_columns -%} + {%- do include_columns.append('[' ~ column | replace(']', ']]') ~ ']') -%} + {%- endfor %} + create + {% if index_config.unique -%} unique {% endif %}{{ index_config.type }} + index [{{ index_name }}] + on {{ relation }} + ({{ key_columns | join(', ') }}) + {% if include_columns -%} + include ({{ include_columns | join(', ') }}) + {% endif %} + {% if index_config.where -%} + where {{ index_config.where }} + {% endif %} + {#- optimize_for_sequential_key, RESUMABLE and RESUMABLE's MAX_DURATION are + CREATE INDEX options that SQL Server only recognizes on 2019 (major 15) + and newer. This adapter still supports 2017/2016, so on older engines + they are dropped (with a warning) and the index is built without them, + rather than failing with "is not a recognized CREATE INDEX option". + ONLINE is intentionally kept: it is recognized on 2017 (edition-gated, + not version-gated). The server version is only queried when one of these + options is actually requested, so the common path adds no round-trip. -#} + {%- set v2019_only_build_options = ['resumable', 'max_duration'] -%} + {%- set _build_options = index_config.build_options or {} -%} + {%- set _wants_v2019_option = index_config.optimize_for_sequential_key + or _build_options.get('resumable') or _build_options.get('max_duration') -%} + {%- set drop_v2019_options = false -%} + {%- if _wants_v2019_option -%} + {%- set _major = sqlserver__server_major_version() -%} + {%- if _major is not none and _major < 15 -%} + {%- set drop_v2019_options = true -%} + {%- do log( + "Index [" ~ index_name ~ "] on " ~ relation ~ ": optimize_for_sequential_key" + ~ " / resumable require SQL Server 2019 (15.x) or newer; building the index" + ~ " without them on detected major version " ~ _major ~ ".", info=true + ) -%} + {%- endif -%} + {%- endif -%} + {%- set with_options = [] -%} + {%- if index_config.data_compression -%} + {%- do with_options.append('data_compression = ' ~ index_config.data_compression | upper) -%} + {%- endif -%} + {%- if index_config.fillfactor -%} + {%- do with_options.append('fillfactor = ' ~ index_config.fillfactor) -%} + {%- endif -%} + {%- if index_config.pad_index -%} + {%- do with_options.append('pad_index = on') -%} + {%- endif -%} + {%- if index_config.ignore_dup_key -%} + {%- do with_options.append('ignore_dup_key = on') -%} + {%- endif -%} + {%- if index_config.optimize_for_sequential_key and not drop_v2019_options -%} + {%- do with_options.append('optimize_for_sequential_key = on') -%} + {%- endif -%} + {%- if index_config.sort_in_tempdb -%} + {%- do with_options.append('sort_in_tempdb = on') -%} + {%- endif -%} + {%- for option_key, option_value in _build_options.items() -%} + {%- if drop_v2019_options and option_key in v2019_only_build_options -%} + {#- skipped: not recognized before SQL Server 2019 -#} + {%- elif option_value is sameas true -%} + {%- do with_options.append(option_key ~ ' = on') -%} + {%- elif option_value is sameas false -%} + {%- do with_options.append(option_key ~ ' = off') -%} + {%- elif option_key == 'max_duration' -%} + {%- do with_options.append('max_duration = ' ~ option_value ~ ' minutes') -%} + {%- else -%} + {%- do with_options.append(option_key ~ ' = ' ~ option_value) -%} + {%- endif -%} + {%- endfor -%} + {% if with_options %} + with ({{ with_options | join(', ') }}) + {% endif %} + end +{%- endmacro %} + + +{% macro sqlserver__describe_indexes(relation) %} + {% call statement('describe_indexes', fetch_result=True) -%} + select + i.[name] as [name], + case when i.[type] = 1 then 'clustered' + when i.[type] = 2 then 'nonclustered' + when i.[type] = 5 then 'clustered columnstore' + when i.[type] = 6 then 'columnstore' + end as [type], + i.is_unique as [unique], + i.is_primary_key as is_primary_key, + i.is_unique_constraint as is_unique_constraint, + isnull(key_cols.cols, '') as [columns], + isnull(incl_cols.cols, '') as included_columns, + case when i.[type] in (1, 2, 6) + then isnull(part.data_compression_desc, 'NONE') + end as data_compression, + isnull(desc_cols.cols, '') as descending_columns, + i.filter_definition as [where], + i.fill_factor as [fillfactor], + i.ignore_dup_key as [ignore_dup_key] + /* optimize_for_sequential_key is deliberately not selected: the + sys.indexes column only exists on SQL Server 2019+ and managed + comparisons are name-based, so it isn't needed here */ + from sys.indexes i {{ information_schema_hints() }} + outer apply ( + /* STRING_AGG ... WITHIN GROUP requires SQL Server 2017+, the floor + of this adapter's CI matrix */ + select string_agg(cast(col.[name] as nvarchar(max)), ', ') within group (order by ic.key_ordinal) as cols + from sys.index_columns ic {{ information_schema_hints() }} + inner join sys.columns col {{ information_schema_hints() }} + on col.object_id = ic.object_id and col.column_id = ic.column_id + where ic.object_id = i.object_id and ic.index_id = i.index_id + and ic.is_included_column = 0 + ) key_cols + outer apply ( + select string_agg(cast(col.[name] as nvarchar(max)), ', ') as cols + from sys.index_columns ic {{ information_schema_hints() }} + inner join sys.columns col {{ information_schema_hints() }} + on col.object_id = ic.object_id and col.column_id = ic.column_id + where ic.object_id = i.object_id and ic.index_id = i.index_id + and ic.is_included_column = 1 + ) incl_cols + outer apply ( + select string_agg(cast(col.[name] as nvarchar(max)), ', ') as cols + from sys.index_columns ic {{ information_schema_hints() }} + inner join sys.columns col {{ information_schema_hints() }} + on col.object_id = ic.object_id and col.column_id = ic.column_id + where ic.object_id = i.object_id and ic.index_id = i.index_id + and ic.is_descending_key = 1 + ) desc_cols + outer apply ( + /* MAX() rather than TOP 1: deterministic if partitions ever carry + mixed compression (the adapter doesn't manage partitioning today) */ + select max(p.data_compression_desc) as data_compression_desc + from sys.partitions p {{ information_schema_hints() }} + where p.object_id = i.object_id and p.index_id = i.index_id + ) part + where i.object_id = OBJECT_ID('{{ relation.schema }}.{{ relation.identifier }}') + and i.index_id > 0 + and i.[type] not in (3, 4, 7) /* xml, spatial, memory-optimized hash */ + {%- endcall %} + {{ return(load_result('describe_indexes').table) }} +{% endmacro %} + + +{% macro sqlserver__get_drop_index_sql(relation, index_name) -%} + drop index [{{ index_name }}] on {{ relation }} +{%- endmacro %} + + +{% macro sqlserver__create_indexes(relation) %} + {#- + Override of the dbt-adapters default to validate the index set as a whole + (at most one clustered; clustered rowstore vs as_columnstore conflict) + before creating anything. as_columnstore is only honored by + create_table_as, so it is irrelevant for seeds despite defaulting true. + -#} + {%- set raw_indexes = config.get('indexes', default=[]) -%} + {%- set materialized = config.get('materialized') -%} + {%- set as_columnstore = config.get('as_columnstore', default=true) + if materialized in ('table', 'incremental', 'snapshot') else false -%} + {%- do adapter.validate_indexes( + raw_indexes, as_columnstore, config.get('drop_unmanaged_indexes', default=false) + ) -%} + {%- for _index_dict in raw_indexes %} + {%- set create_index_sql = get_create_index_sql(relation, _index_dict) -%} + {% if create_index_sql %} + {% do run_query(create_index_sql) %} + {% endif %} + {%- endfor %} +{% endmacro %} + + +{% macro sqlserver__reconcile_indexes(relation) %} + {#- + Converge an existing relation on its configured index set. Called on the + paths where the relation persists across runs (incremental non-full- + refresh, dml table refresh, snapshot updates), where create_indexes alone + would let config changes drift. + -#} + {%- set raw_indexes = config.get('indexes', default=[]) -%} + {%- set drop_unmanaged = config.get('drop_unmanaged_indexes', default=false) -%} + {#- all three callers (incremental, dml refresh, snapshot) honor as_columnstore -#} + {%- do adapter.validate_indexes( + raw_indexes, config.get('as_columnstore', default=true), drop_unmanaged + ) -%} + {%- set existing = sqlserver__describe_indexes(relation) -%} + {%- set result = adapter.index_changes(existing, raw_indexes, relation, drop_unmanaged) -%} + {%- for warning in result['warnings'] %} + {% do log("Index reconcile on " ~ relation ~ ": " ~ warning, info=true) %} + {%- endfor %} + {#- Apply all drops and creates in ONE transactional batch: a definition + change is then atomic, with no window where a replacement index (or the + uniqueness it enforces) is missing for concurrent readers. xact_abort + guarantees rollback if any statement fails mid-batch. -#} + {%- set reconcile_statements = [] -%} + {%- for index_name in result['drops'] %} + {% do log("Dropping index " ~ index_name ~ " on " ~ relation, info=true) %} + {%- do reconcile_statements.append(sqlserver__get_drop_index_sql(relation, index_name)) -%} + {%- endfor %} + {%- for index_dict in result['creates'] %} + {%- do reconcile_statements.append(sqlserver__get_create_index_sql(relation, index_dict)) -%} + {%- endfor %} + {% if reconcile_statements %} + {% do run_query( + "set xact_abort on;\nbegin transaction;\n" + ~ reconcile_statements | join(";\n") + ~ ";\ncommit transaction;" + ) %} + {% endif %} + {#- ONLINE / RESUMABLE creates cannot run inside the transaction above + (SQL Server forbids RESUMABLE in a user transaction, and an ONLINE build + wrapped in one holds its locks until commit, negating the non-blocking + intent). Apply them individually in autocommit. The drops above have + already committed, so a replacement index still builds after its + predecessor is gone; these are not part of the atomic batch, so there is + a brief window where the new index is absent for readers. -#} + {%- for index_dict in result['creates_no_txn'] %} + {% do log("Creating ONLINE/RESUMABLE index outside the reconcile transaction on " ~ relation, info=true) %} + {% do run_query(sqlserver__get_create_index_sql(relation, index_dict)) %} + {%- endfor %} +{% endmacro %} diff --git a/tests/functional/adapter/mssql/test_index_macros.py b/tests/functional/adapter/mssql/test_index_macros.py index 15907a182..d8df11d19 100644 --- a/tests/functional/adapter/mssql/test_index_macros.py +++ b/tests/functional/adapter/mssql/test_index_macros.py @@ -1,430 +1,430 @@ -import pytest - -from dbt.tests.util import get_connection, run_dbt -from tests.functional.adapter.mssql.test_index_config import index_count, indexes_def - -# flake8: noqa: E501 - -index_seed_csv = """id_col,data,secondary_data,tertiary_data -1,'a'",122,20 -""" - -index_schema_base_yml = """ -version: 2 -seeds: - - name: raw_data - config: - column_types: - id_col: integer - data: nvarchar(20) - secondary_data: integer - tertiary_data: bigint -""" - -model_yml = """ -version: 2 -models: - - name: index_model - - name: index_ccs_model -""" - -model_sql = """ -{{ - config({ - "materialized": 'table', - "as_columnstore": False, - "post-hook": [ - "{{ create_clustered_index(columns = ['id_col'], unique=True) }}", - "{{ create_nonclustered_index(columns = ['data']) }}", - "{{ create_nonclustered_index(columns = ['secondary_data'], includes = ['tertiary_data']) }}", - ] - }) -}} - select * from {{ ref('raw_data') }} -""" - -model_sql_ccs = """ -{{ - config({ - "materialized": 'table', - "post-hook": [ - "{{ create_nonclustered_index(columns = ['data']) }}", - "{{ create_nonclustered_index(columns = ['secondary_data'], includes = ['tertiary_data']) }}", - ] - }) -}} - select * from {{ ref('raw_data') }} -""" - -drop_schema_model = """ -{{ - config({ - "materialized": 'table', - "post-hook": [ - "{{ drop_all_indexes_on_table() }}", - ] - }) -}} -select * from {{ ref('raw_data') }} -""" - - -class TestIndex: - @pytest.fixture(scope="class") - def project_config_update(self): - return {"name": "generic_tests"} - - @pytest.fixture(scope="class") - def seeds(self): - return { - "raw_data.csv": index_seed_csv, - "schema.yml": index_schema_base_yml, - } - - @pytest.fixture(scope="class") - def models(self): - return { - "index_model.sql": model_sql, - "index_ccs_model.sql": model_sql_ccs, - "schema.yml": model_yml, - } - - def drop_artifacts(self, project): - with get_connection(project.adapter): - project.adapter.execute("DROP TABLE IF EXISTS index_model", fetch=True) - project.adapter.execute("DROP TABLE IF EXISTS index_ccs_model") - - def test_create_index(self, project): - run_dbt(["seed"]) - run_dbt(["run"]) - - with get_connection(project.adapter): - result, table = project.adapter.execute( - index_count.format(schema_name=project.created_schemas[0]), fetch=True - ) - schema_dict = {_[0]: _[1] for _ in table.rows} - expected = { - "clustered columnstore": 1, - "clustered unique": 1, - "nonclustered": 4, - } - self.drop_artifacts(project) - assert schema_dict == expected - - -class TestIndexDropsOnlySchema: - @pytest.fixture(scope="class") - def project_config_update(self): - return {"name": "generic_tests"} - - @pytest.fixture(scope="class") - def seeds(self): - return { - "raw_data.csv": index_seed_csv, - "schema.yml": index_schema_base_yml, - } - - @pytest.fixture(scope="class") - def models(self): - return { - "index_model.sql": drop_schema_model, - "index_ccs_model.sql": model_sql_ccs, - "schema.yml": model_yml, - } - - def create_table_and_index_other_schema(self, project): - _schema = project.test_schema + "other" - create_sql = f""" - USE [{project.database}]; - IF NOT EXISTS (SELECT * FROM sys.schemas WHERE name = '{_schema}') - BEGIN - EXEC('CREATE SCHEMA [{_schema}]') - END - """ - - create_table = f""" - CREATE TABLE {_schema}.index_model ( - IDCOL BIGINT - ) - """ - - create_index = f""" - CREATE INDEX sample_schema ON {_schema}.index_model (IDCOL) - """ - with get_connection(project.adapter): - project.adapter.execute(create_sql, fetch=True) - project.adapter.execute(create_table) - project.adapter.execute(create_index) - - def drop_schema_artifacts(self, project): - _schema = project.test_schema + "other" - drop_index = f"DROP INDEX IF EXISTS sample_schema ON {_schema}.index_model" - drop_table = f"DROP TABLE IF EXISTS {_schema}.index_model" - drop_schema = f"DROP SCHEMA IF EXISTS {_schema}" - - with get_connection(project.adapter): - project.adapter.execute(drop_index, fetch=True) - project.adapter.execute(drop_table) - project.adapter.execute(drop_schema) - - def validate_other_schema(self, project): - with get_connection(project.adapter): - result, table = project.adapter.execute( - indexes_def.format( - schema_name=project.test_schema + "other", table_name="index_model" - ), - fetch=True, - ) - - assert len(table.rows) == 1 - - def test_create_index(self, project): - self.create_table_and_index_other_schema(project) - run_dbt(["seed"]) - run_dbt(["run"]) - self.validate_other_schema(project) - self.drop_schema_artifacts(project) - - -# --------------------------------------------------------------------------- -# Integration tests for the refactored index subsystem. -# -# These run against a live SQL Server (credentials from test.env) and verify: -# * incremental runs produce identical, idempotent indexes -# * INCLUDE columns are registered as is_included_column = 1 -# * quoted / weird identifiers are safely bracket-escaped -# * schema-qualified models keep indexes isolated per schema -# * multi-threaded runs are race-free (deterministic names + IF NOT EXISTS) -# * CCI is created on the final relation, not the __dbt_tmp intermediate -# * CCI name on the final table never contains __dbt_tmp or __dbt_backup -# --------------------------------------------------------------------------- - - -index_columns_for_table = """ -SELECT - i.[name] AS index_name, - i.type_desc AS index_type, - c.[name] AS column_name, - ic.is_included_column AS is_included, - ic.key_ordinal AS key_ordinal -FROM sys.indexes i -INNER JOIN sys.index_columns ic - ON i.object_id = ic.object_id AND i.index_id = ic.index_id -INNER JOIN sys.columns c - ON c.object_id = ic.object_id AND c.column_id = ic.column_id -WHERE i.object_id = OBJECT_ID(N'[{database}].[{schema}].[{table}]') -ORDER BY i.[name], ic.key_ordinal, ic.is_included_column, c.[name] -""" - - -incremental_model_sql = """ -{{ - config({ - "materialized": "incremental", - "unique_key": "id_col", - "as_columnstore": False, - "post-hook": [ - "{{ create_clustered_index(columns=['id_col'], unique=True) }}", - "{{ create_nonclustered_index(columns=['data']) }}", - ], - }) -}} -select * from {{ ref('raw_data') }} -{% if is_incremental() %} -where id_col > (select coalesce(max(id_col), 0) from {{ this }}) -{% endif %} -""" - - -class TestIndexIncremental: - """Indexes should survive incremental runs (idempotent IF NOT EXISTS).""" - - @pytest.fixture(scope="class") - def seeds(self): - return {"raw_data.csv": index_seed_csv, "schema.yml": index_schema_base_yml} - - @pytest.fixture(scope="class") - def models(self): - return {"inc_model.sql": incremental_model_sql} - - def _index_rows(self, project, table_name): - sql = index_columns_for_table.format( - database=project.database, - schema=project.test_schema, - table=table_name, - ) - with get_connection(project.adapter): - _, table = project.adapter.execute(sql, fetch=True) - return list(table.rows) - - def test_indexes_stable_across_incremental_runs(self, project): - run_dbt(["seed"]) - run_dbt(["run"]) - first = self._index_rows(project, "inc_model") - - run_dbt(["run"]) - second = self._index_rows(project, "inc_model") - - assert first == second - index_names = {r[0] for r in second} - assert len(index_names) == 2 - - -include_columns_model_sql = """ -{{ - config({ - "materialized": "table", - "as_columnstore": False, - "post-hook": [ - "{{ create_nonclustered_index(columns=['secondary_data'], includes=['tertiary_data','data']) }}", - ], - }) -}} -select * from {{ ref('raw_data') }} -""" - - -class TestIndexIncludeColumns: - """INCLUDE columns must end up as is_included_column = 1 in sys.index_columns.""" - - @pytest.fixture(scope="class") - def seeds(self): - return {"raw_data.csv": index_seed_csv, "schema.yml": index_schema_base_yml} - - @pytest.fixture(scope="class") - def models(self): - return {"inc_cols_model.sql": include_columns_model_sql} - - def test_include_columns_present(self, project): - run_dbt(["seed"]) - run_dbt(["run"]) - sql = index_columns_for_table.format( - database=project.database, - schema=project.test_schema, - table="inc_cols_model", - ) - with get_connection(project.adapter): - _, table = project.adapter.execute(sql, fetch=True) - rows = list(table.rows) - keyed = [r for r in rows if r[3] == 0] - included = [r for r in rows if r[3] == 1] - assert {r[2] for r in keyed} == {"secondary_data"} - assert {r[2] for r in included} == {"tertiary_data", "data"} - - -quoted_seed_csv = """id col],weird name -1,a -2,b -""" - -quoted_schema_yml = """ -version: 2 -seeds: - - name: raw_data - config: - column_types: - "id col]": integer - "weird name": nvarchar(20) -""" - -quoted_model_sql = """ -{{ - config({ - "materialized": "table", - "as_columnstore": False, - "post-hook": [ - "{{ create_nonclustered_index(columns=['id col]', 'weird name']) }}", - ], - }) -}} -select * from {{ ref('raw_data') }} -""" - - -class TestIndexQuotedIdentifiers: - """Bracket and space characters in column names must be safely quoted.""" - - @pytest.fixture(scope="class") - def seeds(self): - return {"raw_data.csv": quoted_seed_csv, "schema.yml": quoted_schema_yml} - - @pytest.fixture(scope="class") - def models(self): - return {"quoted_model.sql": quoted_model_sql} - - def test_quoted_columns_indexed(self, project): - run_dbt(["seed"]) - run_dbt(["run"]) - sql = index_columns_for_table.format( - database=project.database, - schema=project.test_schema, - table="quoted_model", - ) - with get_connection(project.adapter): - _, table = project.adapter.execute(sql, fetch=True) - col_names = {r[2] for r in table.rows} - assert "id col]" in col_names - assert "weird name" in col_names - - -orphan_cci_check_sql = """ -SELECT COUNT(*) -FROM sys.indexes i -INNER JOIN sys.tables t ON i.object_id = t.object_id -INNER JOIN sys.schemas s ON t.schema_id = s.schema_id -WHERE s.name = '{schema}' - AND (t.name LIKE '%__dbt_tmp' OR t.name LIKE '%__dbt_backup') - AND i.type IN (5, 6) -""" - - -class TestNoOrphanColumnstoreIndex: - """Regression: CCI must be created on the *final* relation, not the - __dbt_tmp intermediate. After a successful run there should be zero CCIs - sitting on any tmp / backup relation in the test schema.""" - - @pytest.fixture(scope="class") - def seeds(self): - return {"raw_data.csv": index_seed_csv, "schema.yml": index_schema_base_yml} - - @pytest.fixture(scope="class") - def models(self): - return { - "cci_model.sql": ( - "{{ config(materialized='table', as_columnstore=True) }}\n" - "select * from {{ ref('raw_data') }}\n" - ) - } - - def test_no_orphan_cci(self, project): - run_dbt(["seed"]) - run_dbt(["run"]) - run_dbt(["run"]) - - with get_connection(project.adapter): - _, t = project.adapter.execute( - orphan_cci_check_sql.format(schema=project.test_schema), - fetch=True, - ) - assert list(t.rows)[0][0] == 0, "Orphaned CCI found on __dbt_tmp/__dbt_backup" - - cci_name_sql = """ - SELECT i.name - FROM sys.indexes i - INNER JOIN sys.tables t ON i.object_id = t.object_id - INNER JOIN sys.schemas s ON t.schema_id = s.schema_id - WHERE s.name = '{schema}' - AND t.name = 'cci_model' - AND i.type IN (5, 6) - """.format(schema=project.test_schema) - with get_connection(project.adapter): - _, t = project.adapter.execute(cci_name_sql, fetch=True) - cci_rows = list(t.rows) - assert len(cci_rows) == 1, f"Expected exactly 1 CCI on cci_model, got {cci_rows}" - cci_name = cci_rows[0][0] - assert ( - "__dbt_tmp" not in cci_name - ), f"CCI name '{cci_name}' contains __dbt_tmp - index would be orphaned after rename" - assert ( - "__dbt_backup" not in cci_name - ), f"CCI name '{cci_name}' contains __dbt_backup - index would be orphaned after rename" +import pytest + +from dbt.tests.util import get_connection, run_dbt +from tests.functional.adapter.mssql.test_index_config import index_count, indexes_def + +# flake8: noqa: E501 + +index_seed_csv = """id_col,data,secondary_data,tertiary_data +1,'a'",122,20 +""" + +index_schema_base_yml = """ +version: 2 +seeds: + - name: raw_data + config: + column_types: + id_col: integer + data: nvarchar(20) + secondary_data: integer + tertiary_data: bigint +""" + +model_yml = """ +version: 2 +models: + - name: index_model + - name: index_ccs_model +""" + +model_sql = """ +{{ + config({ + "materialized": 'table', + "as_columnstore": False, + "post-hook": [ + "{{ create_clustered_index(columns = ['id_col'], unique=True) }}", + "{{ create_nonclustered_index(columns = ['data']) }}", + "{{ create_nonclustered_index(columns = ['secondary_data'], includes = ['tertiary_data']) }}", + ] + }) +}} + select * from {{ ref('raw_data') }} +""" + +model_sql_ccs = """ +{{ + config({ + "materialized": 'table', + "post-hook": [ + "{{ create_nonclustered_index(columns = ['data']) }}", + "{{ create_nonclustered_index(columns = ['secondary_data'], includes = ['tertiary_data']) }}", + ] + }) +}} + select * from {{ ref('raw_data') }} +""" + +drop_schema_model = """ +{{ + config({ + "materialized": 'table', + "post-hook": [ + "{{ drop_all_indexes_on_table() }}", + ] + }) +}} +select * from {{ ref('raw_data') }} +""" + + +class TestIndex: + @pytest.fixture(scope="class") + def project_config_update(self): + return {"name": "generic_tests"} + + @pytest.fixture(scope="class") + def seeds(self): + return { + "raw_data.csv": index_seed_csv, + "schema.yml": index_schema_base_yml, + } + + @pytest.fixture(scope="class") + def models(self): + return { + "index_model.sql": model_sql, + "index_ccs_model.sql": model_sql_ccs, + "schema.yml": model_yml, + } + + def drop_artifacts(self, project): + with get_connection(project.adapter): + project.adapter.execute("DROP TABLE IF EXISTS index_model", fetch=True) + project.adapter.execute("DROP TABLE IF EXISTS index_ccs_model") + + def test_create_index(self, project): + run_dbt(["seed"]) + run_dbt(["run"]) + + with get_connection(project.adapter): + result, table = project.adapter.execute( + index_count.format(schema_name=project.created_schemas[0]), fetch=True + ) + schema_dict = {_[0]: _[1] for _ in table.rows} + expected = { + "clustered columnstore": 1, + "clustered unique": 1, + "nonclustered": 4, + } + self.drop_artifacts(project) + assert schema_dict == expected + + +class TestIndexDropsOnlySchema: + @pytest.fixture(scope="class") + def project_config_update(self): + return {"name": "generic_tests"} + + @pytest.fixture(scope="class") + def seeds(self): + return { + "raw_data.csv": index_seed_csv, + "schema.yml": index_schema_base_yml, + } + + @pytest.fixture(scope="class") + def models(self): + return { + "index_model.sql": drop_schema_model, + "index_ccs_model.sql": model_sql_ccs, + "schema.yml": model_yml, + } + + def create_table_and_index_other_schema(self, project): + _schema = project.test_schema + "other" + create_sql = f""" + USE [{project.database}]; + IF NOT EXISTS (SELECT * FROM sys.schemas WHERE name = '{_schema}') + BEGIN + EXEC('CREATE SCHEMA [{_schema}]') + END + """ + + create_table = f""" + CREATE TABLE {_schema}.index_model ( + IDCOL BIGINT + ) + """ + + create_index = f""" + CREATE INDEX sample_schema ON {_schema}.index_model (IDCOL) + """ + with get_connection(project.adapter): + project.adapter.execute(create_sql, fetch=True) + project.adapter.execute(create_table) + project.adapter.execute(create_index) + + def drop_schema_artifacts(self, project): + _schema = project.test_schema + "other" + drop_index = f"DROP INDEX IF EXISTS sample_schema ON {_schema}.index_model" + drop_table = f"DROP TABLE IF EXISTS {_schema}.index_model" + drop_schema = f"DROP SCHEMA IF EXISTS {_schema}" + + with get_connection(project.adapter): + project.adapter.execute(drop_index, fetch=True) + project.adapter.execute(drop_table) + project.adapter.execute(drop_schema) + + def validate_other_schema(self, project): + with get_connection(project.adapter): + result, table = project.adapter.execute( + indexes_def.format( + schema_name=project.test_schema + "other", table_name="index_model" + ), + fetch=True, + ) + + assert len(table.rows) == 1 + + def test_create_index(self, project): + self.create_table_and_index_other_schema(project) + run_dbt(["seed"]) + run_dbt(["run"]) + self.validate_other_schema(project) + self.drop_schema_artifacts(project) + + +# --------------------------------------------------------------------------- +# Integration tests for the refactored index subsystem. +# +# These run against a live SQL Server (credentials from test.env) and verify: +# * incremental runs produce identical, idempotent indexes +# * INCLUDE columns are registered as is_included_column = 1 +# * quoted / weird identifiers are safely bracket-escaped +# * schema-qualified models keep indexes isolated per schema +# * multi-threaded runs are race-free (deterministic names + IF NOT EXISTS) +# * CCI is created on the final relation, not the __dbt_tmp intermediate +# * CCI name on the final table never contains __dbt_tmp or __dbt_backup +# --------------------------------------------------------------------------- + + +index_columns_for_table = """ +SELECT + i.[name] AS index_name, + i.type_desc AS index_type, + c.[name] AS column_name, + ic.is_included_column AS is_included, + ic.key_ordinal AS key_ordinal +FROM sys.indexes i +INNER JOIN sys.index_columns ic + ON i.object_id = ic.object_id AND i.index_id = ic.index_id +INNER JOIN sys.columns c + ON c.object_id = ic.object_id AND c.column_id = ic.column_id +WHERE i.object_id = OBJECT_ID(N'[{database}].[{schema}].[{table}]') +ORDER BY i.[name], ic.key_ordinal, ic.is_included_column, c.[name] +""" + + +incremental_model_sql = """ +{{ + config({ + "materialized": "incremental", + "unique_key": "id_col", + "as_columnstore": False, + "post-hook": [ + "{{ create_clustered_index(columns=['id_col'], unique=True) }}", + "{{ create_nonclustered_index(columns=['data']) }}", + ], + }) +}} +select * from {{ ref('raw_data') }} +{% if is_incremental() %} +where id_col > (select coalesce(max(id_col), 0) from {{ this }}) +{% endif %} +""" + + +class TestIndexIncremental: + """Indexes should survive incremental runs (idempotent IF NOT EXISTS).""" + + @pytest.fixture(scope="class") + def seeds(self): + return {"raw_data.csv": index_seed_csv, "schema.yml": index_schema_base_yml} + + @pytest.fixture(scope="class") + def models(self): + return {"inc_model.sql": incremental_model_sql} + + def _index_rows(self, project, table_name): + sql = index_columns_for_table.format( + database=project.database, + schema=project.test_schema, + table=table_name, + ) + with get_connection(project.adapter): + _, table = project.adapter.execute(sql, fetch=True) + return list(table.rows) + + def test_indexes_stable_across_incremental_runs(self, project): + run_dbt(["seed"]) + run_dbt(["run"]) + first = self._index_rows(project, "inc_model") + + run_dbt(["run"]) + second = self._index_rows(project, "inc_model") + + assert first == second + index_names = {r[0] for r in second} + assert len(index_names) == 2 + + +include_columns_model_sql = """ +{{ + config({ + "materialized": "table", + "as_columnstore": False, + "post-hook": [ + "{{ create_nonclustered_index(columns=['secondary_data'], includes=['tertiary_data','data']) }}", + ], + }) +}} +select * from {{ ref('raw_data') }} +""" + + +class TestIndexIncludeColumns: + """INCLUDE columns must end up as is_included_column = 1 in sys.index_columns.""" + + @pytest.fixture(scope="class") + def seeds(self): + return {"raw_data.csv": index_seed_csv, "schema.yml": index_schema_base_yml} + + @pytest.fixture(scope="class") + def models(self): + return {"inc_cols_model.sql": include_columns_model_sql} + + def test_include_columns_present(self, project): + run_dbt(["seed"]) + run_dbt(["run"]) + sql = index_columns_for_table.format( + database=project.database, + schema=project.test_schema, + table="inc_cols_model", + ) + with get_connection(project.adapter): + _, table = project.adapter.execute(sql, fetch=True) + rows = list(table.rows) + keyed = [r for r in rows if r[3] == 0] + included = [r for r in rows if r[3] == 1] + assert {r[2] for r in keyed} == {"secondary_data"} + assert {r[2] for r in included} == {"tertiary_data", "data"} + + +quoted_seed_csv = """id col],weird name +1,a +2,b +""" + +quoted_schema_yml = """ +version: 2 +seeds: + - name: raw_data + config: + column_types: + "id col]": integer + "weird name": nvarchar(20) +""" + +quoted_model_sql = """ +{{ + config({ + "materialized": "table", + "as_columnstore": False, + "post-hook": [ + "{{ create_nonclustered_index(columns=['id col]', 'weird name']) }}", + ], + }) +}} +select * from {{ ref('raw_data') }} +""" + + +class TestIndexQuotedIdentifiers: + """Bracket and space characters in column names must be safely quoted.""" + + @pytest.fixture(scope="class") + def seeds(self): + return {"raw_data.csv": quoted_seed_csv, "schema.yml": quoted_schema_yml} + + @pytest.fixture(scope="class") + def models(self): + return {"quoted_model.sql": quoted_model_sql} + + def test_quoted_columns_indexed(self, project): + run_dbt(["seed"]) + run_dbt(["run"]) + sql = index_columns_for_table.format( + database=project.database, + schema=project.test_schema, + table="quoted_model", + ) + with get_connection(project.adapter): + _, table = project.adapter.execute(sql, fetch=True) + col_names = {r[2] for r in table.rows} + assert "id col]" in col_names + assert "weird name" in col_names + + +orphan_cci_check_sql = """ +SELECT COUNT(*) +FROM sys.indexes i +INNER JOIN sys.tables t ON i.object_id = t.object_id +INNER JOIN sys.schemas s ON t.schema_id = s.schema_id +WHERE s.name = '{schema}' + AND (t.name LIKE '%__dbt_tmp' OR t.name LIKE '%__dbt_backup') + AND i.type IN (5, 6) +""" + + +class TestNoOrphanColumnstoreIndex: + """Regression: CCI must be created on the *final* relation, not the + __dbt_tmp intermediate. After a successful run there should be zero CCIs + sitting on any tmp / backup relation in the test schema.""" + + @pytest.fixture(scope="class") + def seeds(self): + return {"raw_data.csv": index_seed_csv, "schema.yml": index_schema_base_yml} + + @pytest.fixture(scope="class") + def models(self): + return { + "cci_model.sql": ( + "{{ config(materialized='table', as_columnstore=True) }}\n" + "select * from {{ ref('raw_data') }}\n" + ) + } + + def test_no_orphan_cci(self, project): + run_dbt(["seed"]) + run_dbt(["run"]) + run_dbt(["run"]) + + with get_connection(project.adapter): + _, t = project.adapter.execute( + orphan_cci_check_sql.format(schema=project.test_schema), + fetch=True, + ) + assert list(t.rows)[0][0] == 0, "Orphaned CCI found on __dbt_tmp/__dbt_backup" + + cci_name_sql = """ + SELECT i.name + FROM sys.indexes i + INNER JOIN sys.tables t ON i.object_id = t.object_id + INNER JOIN sys.schemas s ON t.schema_id = s.schema_id + WHERE s.name = '{schema}' + AND t.name = 'cci_model' + AND i.type IN (5, 6) + """.format(schema=project.test_schema) + with get_connection(project.adapter): + _, t = project.adapter.execute(cci_name_sql, fetch=True) + cci_rows = list(t.rows) + assert len(cci_rows) == 1, f"Expected exactly 1 CCI on cci_model, got {cci_rows}" + cci_name = cci_rows[0][0] + assert ( + "__dbt_tmp" not in cci_name + ), f"CCI name '{cci_name}' contains __dbt_tmp - index would be orphaned after rename" + assert ( + "__dbt_backup" not in cci_name + ), f"CCI name '{cci_name}' contains __dbt_backup - index would be orphaned after rename" diff --git a/tests/unit/adapters/mssql/test_indexes.py b/tests/unit/adapters/mssql/test_indexes.py index 189ef0c06..21b09298b 100644 --- a/tests/unit/adapters/mssql/test_indexes.py +++ b/tests/unit/adapters/mssql/test_indexes.py @@ -1,338 +1,338 @@ -""" -Unit tests for the SQL Server index macros. - -Mirrors the style of test_generate_schema_name.py: inline Jinja, no DB. - -Coverage: - * mssql__quote_ident - bracket escaping - * mssql__qualified_relation - 3-part name assembly - * mssql__strip_dbt_suffix - __dbt_tmp / __dbt_backup stripping - * mssql__index_name - determinism, 116-char cap, hashing, - immunity to __dbt_tmp suffix - * sqlserver__index_exists - emitted SQL shape (OBJECT_ID scoped) - * create_clustered_index - quoted columns, IF NOT EXISTS wrapper - * create_nonclustered_index - INCLUDE columns, quoting, idempotence -""" - -import hashlib -import re -from pathlib import Path - -import jinja2 -import pytest -from jinja2.runtime import Macro as _Jinja2Macro - - -class _MacroReturn(BaseException): - def __init__(self, value): - self.value = value - - -_orig_macro_call = _Jinja2Macro.__call__ - - -def _patched_macro_call(self, *args, **kwargs): - try: - return _orig_macro_call(self, *args, **kwargs) - except _MacroReturn as exc: - return exc.value - - -@pytest.fixture(scope="module", autouse=True) -def _patch_jinja2_macro_return(): - _Jinja2Macro.__call__ = _patched_macro_call - yield - _Jinja2Macro.__call__ = _orig_macro_call - - -MACRO_PATH = ( - Path(__file__).resolve().parents[4] - / "dbt" - / "include" - / "sqlserver" - / "macros" - / "adapters" - / "indexes.sql" -) - -MACRO_SRC = MACRO_PATH.read_text(encoding="utf-8") - - -class _FakeRelation: - """Minimal stand-in for dbt's Relation object.""" - - def __init__(self, database, schema, identifier): - self.database = database - self.schema = schema - self.identifier = identifier - - def __str__(self): - return f"[{self.database}].[{self.schema}].[{self.identifier}]" - - -def _env(): - env = jinja2.Environment( - trim_blocks=True, - lstrip_blocks=True, - extensions=["jinja2.ext.do"], - ) - - def local_md5(s): - return hashlib.md5(s.encode("utf-8")).hexdigest() - - env.globals.update( - { - "local_md5": local_md5, - "information_schema_hints": lambda: "", - "log": lambda *a, **kw: "", - "get_use_database_sql": lambda db: f"USE [{db}];", - "this": _FakeRelation("mydb", "myschema", "my_model"), - "return": lambda v: (_ for _ in ()).throw(_MacroReturn(v)), - } - ) - return env - - -def _render(call_expr, **ctx): - env = _env() - template = env.from_string(MACRO_SRC + "\n" + "{{ " + call_expr + " }}") - return template.render(**ctx).strip() - - -def _normalize_ws(s): - return re.sub(r"\s+", " ", s).strip() - - -class TestQuoteIdent: - def test_simple_name(self): - assert _render("mssql__quote_ident('foo')") == "[foo]" - - def test_name_with_space(self): - assert _render("mssql__quote_ident('weird name')") == "[weird name]" - - def test_name_with_close_bracket_is_escaped(self): - assert _render("mssql__quote_ident('we][ird')") == "[we]][ird]" - - def test_name_with_dot_is_not_split(self): - assert _render("mssql__quote_ident('a.b')") == "[a.b]" - - -class TestQualifiedRelation: - def test_three_part_name(self): - rel = _FakeRelation("mydb", "myschema", "my_model") - out = _render("mssql__qualified_relation(rel)", rel=rel) - assert out == "[mydb].[myschema].[my_model]" - - def test_escapes_brackets_in_every_part(self): - rel = _FakeRelation("d]b", "sch]ema", "id]ent") - out = _render("mssql__qualified_relation(rel)", rel=rel) - assert out == "[d]]b].[sch]]ema].[id]]ent]" - - -class TestStripDbtSuffix: - @pytest.mark.parametrize( - "identifier, expected", - [ - ("my_model", "my_model"), - ("my_model__dbt_tmp", "my_model"), - ("my_model__dbt_backup", "my_model"), - ("my_model__dbt_tmp_vw", "my_model"), - ("my__dbt_tmp_model", "my__dbt_tmp_model"), # __dbt_tmp in middle, not a suffix - ], - ) - def test_strip(self, identifier, expected): - assert _render(f"mssql__strip_dbt_suffix('{identifier}')") == expected - - -class TestIndexName: - def test_deterministic(self): - rel = _FakeRelation("d", "s", "t") - a = _render("mssql__index_name(rel, 'nidx', ['x','y'])", rel=rel) - b = _render("mssql__index_name(rel, 'nidx', ['x','y'])", rel=rel) - assert a == b - - def test_column_order_changes_name(self): - rel = _FakeRelation("d", "s", "t") - a = _render("mssql__index_name(rel, 'nidx', ['x','y'])", rel=rel) - b = _render("mssql__index_name(rel, 'nidx', ['y','x'])", rel=rel) - assert a != b - - def test_unique_flag_changes_name(self): - rel = _FakeRelation("d", "s", "t") - a = _render("mssql__index_name(rel, 'cidx', ['x'], unique=False)", rel=rel) - b = _render("mssql__index_name(rel, 'cidx', ['x'], unique=True)", rel=rel) - assert a != b - - def test_includes_changes_name(self): - rel = _FakeRelation("d", "s", "t") - a = _render("mssql__index_name(rel, 'nidx', ['x'])", rel=rel) - b = _render("mssql__index_name(rel, 'nidx', ['x'], includes=['y'])", rel=rel) - assert a != b - - def test_accepts_string_columns(self): - rel = _FakeRelation("d", "s", "t") - name = _render("mssql__index_name(rel, 'nidx', 'x')", rel=rel) - assert name.startswith("nidx_t_") - - def test_false_includes_are_treated_as_empty(self): - rel = _FakeRelation("d", "s", "t") - a = _render("mssql__index_name(rel, 'nidx', ['x'], includes=false)", rel=rel) - b = _render("mssql__index_name(rel, 'nidx', ['x'])", rel=rel) - assert a == b - - def test_dbt_tmp_suffix_does_not_affect_name(self): - a = _render( - "mssql__index_name(rel, 'cci', ['__all__'])", - rel=_FakeRelation("d", "s", "my_model"), - ) - b = _render( - "mssql__index_name(rel, 'cci', ['__all__'])", - rel=_FakeRelation("d", "s", "my_model__dbt_tmp"), - ) - c = _render( - "mssql__index_name(rel, 'cci', ['__all__'])", - rel=_FakeRelation("d", "s", "my_model__dbt_backup"), - ) - assert a == b == c - - def test_length_capped_at_116_chars(self): - rel = _FakeRelation("d", "s", "x" * 500) - name = _render("mssql__index_name(rel, 'nidx', ['c'])", rel=rel) - assert len(name) <= 116 - - def test_long_name_still_unique_per_signature(self): - rel = _FakeRelation("d", "s", "x" * 500) - a = _render("mssql__index_name(rel, 'nidx', ['c1'])", rel=rel) - b = _render("mssql__index_name(rel, 'nidx', ['c2'])", rel=rel) - assert a != b - assert len(a) <= 116 - assert len(b) <= 116 - - def test_brackets_stripped_from_readable_part(self): - rel = _FakeRelation("d", "s", "[weird]") - name = _render("mssql__index_name(rel, 'nidx', ['c'])", rel=rel) - assert "[" not in name and "]" not in name - - -class TestIndexExists: - def test_emits_object_id_scoped_check(self): - rel = _FakeRelation("mydb", "myschema", "my_model") - sql = _render("sqlserver__index_exists(rel, 'idx_foo')", rel=rel) - sql = _normalize_ws(sql) - assert "EXISTS" in sql - assert "sys.indexes" in sql - assert "name = N'idx_foo'" in sql - assert "OBJECT_ID(N'[mydb].[myschema].[my_model]')" in sql - - -class TestCreateClusteredIndex: - def test_quotes_columns(self): - rel = _FakeRelation("d", "s", "t") - sql = _render("create_clustered_index(['id_col', 'data'], relation=rel)", rel=rel) - sql = _normalize_ws(sql) - assert "([id_col], [data])" in sql - - def test_wrapped_in_if_not_exists(self): - rel = _FakeRelation("d", "s", "t") - sql = _normalize_ws(_render("create_clustered_index(['c'], relation=rel)", rel=rel)) - assert "if not exists" in sql - assert "begin create" in sql - assert sql.endswith("end") - - def test_unique_flag_emits_unique_keyword(self): - rel = _FakeRelation("d", "s", "t") - sql = _normalize_ws( - _render("create_clustered_index(['c'], unique=True, relation=rel)", rel=rel) - ) - assert "unique clustered index" in sql - - def test_accepts_string_column(self): - rel = _FakeRelation("d", "s", "t") - sql = _normalize_ws(_render("create_clustered_index('id_col', relation=rel)", rel=rel)) - assert "([id_col])" in sql - - -class TestCreateNonclusteredIndex: - def test_emits_nonclustered(self): - rel = _FakeRelation("d", "s", "t") - sql = _normalize_ws(_render("create_nonclustered_index(['c'], relation=rel)", rel=rel)) - assert "create nonclustered index" in sql - - def test_accepts_string_column(self): - rel = _FakeRelation("d", "s", "t") - sql = _normalize_ws(_render("create_nonclustered_index('c', relation=rel)", rel=rel)) - assert "([c])" in sql - - def test_accepts_string_include_column(self): - rel = _FakeRelation("d", "s", "t") - sql = _normalize_ws( - _render("create_nonclustered_index(['c'], includes='inc1', relation=rel)", rel=rel) - ) - assert "include ([inc1])" in sql - - def test_include_columns_quoted(self): - rel = _FakeRelation("d", "s", "t") - sql = _normalize_ws( - _render( - "create_nonclustered_index(['c'], includes=['inc1','inc2'], relation=rel)", - rel=rel, - ) - ) - assert "include ([inc1], [inc2])" in sql - - def test_no_include_block_when_no_includes(self): - rel = _FakeRelation("d", "s", "t") - sql = _normalize_ws(_render("create_nonclustered_index(['c'], relation=rel)", rel=rel)) - assert "include" not in sql - - def test_false_include_is_treated_as_empty(self): - rel = _FakeRelation("d", "s", "t") - sql = _normalize_ws( - _render("create_nonclustered_index(['c'], includes=false, relation=rel)", rel=rel) - ) - assert "include" not in sql - - def test_columns_with_brackets_are_escaped(self): - rel = _FakeRelation("d", "s", "t") - sql = _normalize_ws( - _render("create_nonclustered_index(['we]ird'], relation=rel)", rel=rel) - ) - assert "[we]]ird]" in sql - - def test_idempotent_wrapper(self): - rel = _FakeRelation("d", "s", "t") - sql = _normalize_ws(_render("create_nonclustered_index(['c'], relation=rel)", rel=rel)) - assert "if not exists" in sql and "begin" in sql and "end" in sql - - -class TestColumnstoreIndexName: - def test_intermediate_relation_uses_target_name(self): - intermediate = _FakeRelation("d", "s", "my_model__dbt_tmp") - target = _FakeRelation("d", "s", "my_model") - sql_int = _render( - "sqlserver__create_clustered_columnstore_index(intermediate)", - intermediate=intermediate, - ) - sql_final = _render( - "sqlserver__create_clustered_columnstore_index(target)", - target=target, - ) - name_re = re.compile( - r"CREATE\s+CLUSTERED\s+COLUMNSTORE\s+INDEX\s+(\[[^\]]+\])", re.IGNORECASE - ) - name_int = name_re.search(_normalize_ws(sql_int)).group(1) - name_final = name_re.search(_normalize_ws(sql_final)).group(1) - assert name_int == name_final, ( - f"intermediate CCI name {name_int} differs from final {name_final}; " - "would be orphaned after rename" - ) - - def test_uses_qualified_target_relation_for_create(self): - target = _FakeRelation("mydb", "myschema", "my_model") - sql = _normalize_ws( - _render( - "sqlserver__create_clustered_columnstore_index(target)", - target=target, - ) - ) - assert "ON [mydb].[myschema].[my_model]" in sql +""" +Unit tests for the SQL Server index macros. + +Mirrors the style of test_generate_schema_name.py: inline Jinja, no DB. + +Coverage: + * mssql__quote_ident - bracket escaping + * mssql__qualified_relation - 3-part name assembly + * mssql__strip_dbt_suffix - __dbt_tmp / __dbt_backup stripping + * mssql__index_name - determinism, 116-char cap, hashing, + immunity to __dbt_tmp suffix + * sqlserver__index_exists - emitted SQL shape (OBJECT_ID scoped) + * create_clustered_index - quoted columns, IF NOT EXISTS wrapper + * create_nonclustered_index - INCLUDE columns, quoting, idempotence +""" + +import hashlib +import re +from pathlib import Path + +import jinja2 +import pytest +from jinja2.runtime import Macro as _Jinja2Macro + + +class _MacroReturn(BaseException): + def __init__(self, value): + self.value = value + + +_orig_macro_call = _Jinja2Macro.__call__ + + +def _patched_macro_call(self, *args, **kwargs): + try: + return _orig_macro_call(self, *args, **kwargs) + except _MacroReturn as exc: + return exc.value + + +@pytest.fixture(scope="module", autouse=True) +def _patch_jinja2_macro_return(): + _Jinja2Macro.__call__ = _patched_macro_call + yield + _Jinja2Macro.__call__ = _orig_macro_call + + +MACRO_PATH = ( + Path(__file__).resolve().parents[4] + / "dbt" + / "include" + / "sqlserver" + / "macros" + / "adapters" + / "indexes.sql" +) + +MACRO_SRC = MACRO_PATH.read_text(encoding="utf-8") + + +class _FakeRelation: + """Minimal stand-in for dbt's Relation object.""" + + def __init__(self, database, schema, identifier): + self.database = database + self.schema = schema + self.identifier = identifier + + def __str__(self): + return f"[{self.database}].[{self.schema}].[{self.identifier}]" + + +def _env(): + env = jinja2.Environment( + trim_blocks=True, + lstrip_blocks=True, + extensions=["jinja2.ext.do"], + ) + + def local_md5(s): + return hashlib.md5(s.encode("utf-8")).hexdigest() + + env.globals.update( + { + "local_md5": local_md5, + "information_schema_hints": lambda: "", + "log": lambda *a, **kw: "", + "get_use_database_sql": lambda db: f"USE [{db}];", + "this": _FakeRelation("mydb", "myschema", "my_model"), + "return": lambda v: (_ for _ in ()).throw(_MacroReturn(v)), + } + ) + return env + + +def _render(call_expr, **ctx): + env = _env() + template = env.from_string(MACRO_SRC + "\n" + "{{ " + call_expr + " }}") + return template.render(**ctx).strip() + + +def _normalize_ws(s): + return re.sub(r"\s+", " ", s).strip() + + +class TestQuoteIdent: + def test_simple_name(self): + assert _render("mssql__quote_ident('foo')") == "[foo]" + + def test_name_with_space(self): + assert _render("mssql__quote_ident('weird name')") == "[weird name]" + + def test_name_with_close_bracket_is_escaped(self): + assert _render("mssql__quote_ident('we][ird')") == "[we]][ird]" + + def test_name_with_dot_is_not_split(self): + assert _render("mssql__quote_ident('a.b')") == "[a.b]" + + +class TestQualifiedRelation: + def test_three_part_name(self): + rel = _FakeRelation("mydb", "myschema", "my_model") + out = _render("mssql__qualified_relation(rel)", rel=rel) + assert out == "[mydb].[myschema].[my_model]" + + def test_escapes_brackets_in_every_part(self): + rel = _FakeRelation("d]b", "sch]ema", "id]ent") + out = _render("mssql__qualified_relation(rel)", rel=rel) + assert out == "[d]]b].[sch]]ema].[id]]ent]" + + +class TestStripDbtSuffix: + @pytest.mark.parametrize( + "identifier, expected", + [ + ("my_model", "my_model"), + ("my_model__dbt_tmp", "my_model"), + ("my_model__dbt_backup", "my_model"), + ("my_model__dbt_tmp_vw", "my_model"), + ("my__dbt_tmp_model", "my__dbt_tmp_model"), # __dbt_tmp in middle, not a suffix + ], + ) + def test_strip(self, identifier, expected): + assert _render(f"mssql__strip_dbt_suffix('{identifier}')") == expected + + +class TestIndexName: + def test_deterministic(self): + rel = _FakeRelation("d", "s", "t") + a = _render("mssql__index_name(rel, 'nidx', ['x','y'])", rel=rel) + b = _render("mssql__index_name(rel, 'nidx', ['x','y'])", rel=rel) + assert a == b + + def test_column_order_changes_name(self): + rel = _FakeRelation("d", "s", "t") + a = _render("mssql__index_name(rel, 'nidx', ['x','y'])", rel=rel) + b = _render("mssql__index_name(rel, 'nidx', ['y','x'])", rel=rel) + assert a != b + + def test_unique_flag_changes_name(self): + rel = _FakeRelation("d", "s", "t") + a = _render("mssql__index_name(rel, 'cidx', ['x'], unique=False)", rel=rel) + b = _render("mssql__index_name(rel, 'cidx', ['x'], unique=True)", rel=rel) + assert a != b + + def test_includes_changes_name(self): + rel = _FakeRelation("d", "s", "t") + a = _render("mssql__index_name(rel, 'nidx', ['x'])", rel=rel) + b = _render("mssql__index_name(rel, 'nidx', ['x'], includes=['y'])", rel=rel) + assert a != b + + def test_accepts_string_columns(self): + rel = _FakeRelation("d", "s", "t") + name = _render("mssql__index_name(rel, 'nidx', 'x')", rel=rel) + assert name.startswith("nidx_t_") + + def test_false_includes_are_treated_as_empty(self): + rel = _FakeRelation("d", "s", "t") + a = _render("mssql__index_name(rel, 'nidx', ['x'], includes=false)", rel=rel) + b = _render("mssql__index_name(rel, 'nidx', ['x'])", rel=rel) + assert a == b + + def test_dbt_tmp_suffix_does_not_affect_name(self): + a = _render( + "mssql__index_name(rel, 'cci', ['__all__'])", + rel=_FakeRelation("d", "s", "my_model"), + ) + b = _render( + "mssql__index_name(rel, 'cci', ['__all__'])", + rel=_FakeRelation("d", "s", "my_model__dbt_tmp"), + ) + c = _render( + "mssql__index_name(rel, 'cci', ['__all__'])", + rel=_FakeRelation("d", "s", "my_model__dbt_backup"), + ) + assert a == b == c + + def test_length_capped_at_116_chars(self): + rel = _FakeRelation("d", "s", "x" * 500) + name = _render("mssql__index_name(rel, 'nidx', ['c'])", rel=rel) + assert len(name) <= 116 + + def test_long_name_still_unique_per_signature(self): + rel = _FakeRelation("d", "s", "x" * 500) + a = _render("mssql__index_name(rel, 'nidx', ['c1'])", rel=rel) + b = _render("mssql__index_name(rel, 'nidx', ['c2'])", rel=rel) + assert a != b + assert len(a) <= 116 + assert len(b) <= 116 + + def test_brackets_stripped_from_readable_part(self): + rel = _FakeRelation("d", "s", "[weird]") + name = _render("mssql__index_name(rel, 'nidx', ['c'])", rel=rel) + assert "[" not in name and "]" not in name + + +class TestIndexExists: + def test_emits_object_id_scoped_check(self): + rel = _FakeRelation("mydb", "myschema", "my_model") + sql = _render("sqlserver__index_exists(rel, 'idx_foo')", rel=rel) + sql = _normalize_ws(sql) + assert "EXISTS" in sql + assert "sys.indexes" in sql + assert "name = N'idx_foo'" in sql + assert "OBJECT_ID(N'[mydb].[myschema].[my_model]')" in sql + + +class TestCreateClusteredIndex: + def test_quotes_columns(self): + rel = _FakeRelation("d", "s", "t") + sql = _render("create_clustered_index(['id_col', 'data'], relation=rel)", rel=rel) + sql = _normalize_ws(sql) + assert "([id_col], [data])" in sql + + def test_wrapped_in_if_not_exists(self): + rel = _FakeRelation("d", "s", "t") + sql = _normalize_ws(_render("create_clustered_index(['c'], relation=rel)", rel=rel)) + assert "if not exists" in sql + assert "begin create" in sql + assert sql.endswith("end") + + def test_unique_flag_emits_unique_keyword(self): + rel = _FakeRelation("d", "s", "t") + sql = _normalize_ws( + _render("create_clustered_index(['c'], unique=True, relation=rel)", rel=rel) + ) + assert "unique clustered index" in sql + + def test_accepts_string_column(self): + rel = _FakeRelation("d", "s", "t") + sql = _normalize_ws(_render("create_clustered_index('id_col', relation=rel)", rel=rel)) + assert "([id_col])" in sql + + +class TestCreateNonclusteredIndex: + def test_emits_nonclustered(self): + rel = _FakeRelation("d", "s", "t") + sql = _normalize_ws(_render("create_nonclustered_index(['c'], relation=rel)", rel=rel)) + assert "create nonclustered index" in sql + + def test_accepts_string_column(self): + rel = _FakeRelation("d", "s", "t") + sql = _normalize_ws(_render("create_nonclustered_index('c', relation=rel)", rel=rel)) + assert "([c])" in sql + + def test_accepts_string_include_column(self): + rel = _FakeRelation("d", "s", "t") + sql = _normalize_ws( + _render("create_nonclustered_index(['c'], includes='inc1', relation=rel)", rel=rel) + ) + assert "include ([inc1])" in sql + + def test_include_columns_quoted(self): + rel = _FakeRelation("d", "s", "t") + sql = _normalize_ws( + _render( + "create_nonclustered_index(['c'], includes=['inc1','inc2'], relation=rel)", + rel=rel, + ) + ) + assert "include ([inc1], [inc2])" in sql + + def test_no_include_block_when_no_includes(self): + rel = _FakeRelation("d", "s", "t") + sql = _normalize_ws(_render("create_nonclustered_index(['c'], relation=rel)", rel=rel)) + assert "include" not in sql + + def test_false_include_is_treated_as_empty(self): + rel = _FakeRelation("d", "s", "t") + sql = _normalize_ws( + _render("create_nonclustered_index(['c'], includes=false, relation=rel)", rel=rel) + ) + assert "include" not in sql + + def test_columns_with_brackets_are_escaped(self): + rel = _FakeRelation("d", "s", "t") + sql = _normalize_ws( + _render("create_nonclustered_index(['we]ird'], relation=rel)", rel=rel) + ) + assert "[we]]ird]" in sql + + def test_idempotent_wrapper(self): + rel = _FakeRelation("d", "s", "t") + sql = _normalize_ws(_render("create_nonclustered_index(['c'], relation=rel)", rel=rel)) + assert "if not exists" in sql and "begin" in sql and "end" in sql + + +class TestColumnstoreIndexName: + def test_intermediate_relation_uses_target_name(self): + intermediate = _FakeRelation("d", "s", "my_model__dbt_tmp") + target = _FakeRelation("d", "s", "my_model") + sql_int = _render( + "sqlserver__create_clustered_columnstore_index(intermediate)", + intermediate=intermediate, + ) + sql_final = _render( + "sqlserver__create_clustered_columnstore_index(target)", + target=target, + ) + name_re = re.compile( + r"CREATE\s+CLUSTERED\s+COLUMNSTORE\s+INDEX\s+(\[[^\]]+\])", re.IGNORECASE + ) + name_int = name_re.search(_normalize_ws(sql_int)).group(1) + name_final = name_re.search(_normalize_ws(sql_final)).group(1) + assert name_int == name_final, ( + f"intermediate CCI name {name_int} differs from final {name_final}; " + "would be orphaned after rename" + ) + + def test_uses_qualified_target_relation_for_create(self): + target = _FakeRelation("mydb", "myschema", "my_model") + sql = _normalize_ws( + _render( + "sqlserver__create_clustered_columnstore_index(target)", + target=target, + ) + ) + assert "ON [mydb].[myschema].[my_model]" in sql From ac9c3e784cd6f9104a2b5d275509e8e0b1cdb192 Mon Sep 17 00:00:00 2001 From: Axell Padilla <68310020+axellpadilla@users.noreply.github.com> Date: Thu, 23 Jul 2026 02:00:31 +0000 Subject: [PATCH 5/5] Use Adapter Quoting For SQL Server Indexes y standarize naming and reuse functions --- .../sqlserver/macros/adapters/indexes.sql | 64 +++++-------- tests/unit/adapters/mssql/test_indexes.py | 91 +++++++------------ 2 files changed, 58 insertions(+), 97 deletions(-) diff --git a/dbt/include/sqlserver/macros/adapters/indexes.sql b/dbt/include/sqlserver/macros/adapters/indexes.sql index 49805eb94..9970ee41f 100644 --- a/dbt/include/sqlserver/macros/adapters/indexes.sql +++ b/dbt/include/sqlserver/macros/adapters/indexes.sql @@ -1,18 +1,4 @@ -{% macro mssql__quote_ident(name) -%} - {{ return('[' ~ name | replace(']', ']]') ~ ']') }} -{%- endmacro %} - - -{% macro mssql__qualified_relation(rel) -%} - {{ return( - mssql__quote_ident(rel.database) ~ '.' ~ - mssql__quote_ident(rel.schema) ~ '.' ~ - mssql__quote_ident(rel.identifier) - ) }} -{%- endmacro %} - - -{% macro mssql__strip_dbt_suffix(identifier) -%} +{% macro sqlserver__strip_dbt_suffix(identifier) -%} {%- set ns = namespace(result=identifier) -%} {%- for suffix in ['__dbt_tmp_vw', '__dbt_backup', '__dbt_tmp'] -%} {%- if ns.result.endswith(suffix) -%} @@ -23,10 +9,10 @@ {%- endmacro %} -{% macro mssql__index_name(rel, type, columns, unique=False, includes=false) -%} +{% macro sqlserver__index_name(rel, type, columns, unique=False, includes=false) -%} {%- set cols = [columns] if columns is string else columns -%} {%- set incs = [] if not includes else ([includes] if includes is string else includes) -%} - {%- set stripped = mssql__strip_dbt_suffix(rel.identifier) | replace('[', '') | replace(']', '') -%} + {%- set stripped = sqlserver__strip_dbt_suffix(rel.identifier) | replace('[', '') | replace(']', '') -%} {%- set sig = type ~ '|' ~ (cols | join(',')) ~ '|' ~ (unique | string) ~ '|' ~ (incs | join(',')) -%} {%- set hash = local_md5(sig) -%} {%- set prefix = type ~ '_' ~ stripped -%} @@ -42,27 +28,25 @@ EXISTS ( SELECT 1 FROM sys.indexes {{ information_schema_hints() }} - WHERE name = N'{{ index_name }}' - AND object_id = OBJECT_ID(N'{{ mssql__qualified_relation(rel) }}') + WHERE name = N'{{ escape_single_quotes(index_name) }}' + AND object_id = OBJECT_ID(N'{{ escape_single_quotes(rel) }}') ) {%- endmacro %} {% macro sqlserver__create_clustered_columnstore_index(relation) -%} - {%- set stripped_id = mssql__strip_dbt_suffix(relation.identifier) | replace('[', '') | replace(']', '') -%} + {%- set stripped_id = sqlserver__strip_dbt_suffix(relation.identifier) | replace('[', '') | replace(']', '') -%} {%- set cci_name = relation.schema | replace('[', '') | replace(']', '') ~ '_' ~ stripped_id ~ '_cci' -%} - {%- set cci_literal = cci_name | replace("'", "''") -%} - {%- set full_relation_literal = mssql__qualified_relation(relation) | replace("'", "''") -%} {{ get_use_database_sql(relation.database) }} if EXISTS ( SELECT * FROM sys.indexes {{ information_schema_hints() }} - WHERE name = N'{{ cci_literal }}' - AND object_id = OBJECT_ID(N'{{ full_relation_literal }}') + WHERE name = N'{{ escape_single_quotes(cci_name) }}' + AND object_id = OBJECT_ID(N'{{ escape_single_quotes(relation) }}') ) - DROP INDEX {{ mssql__quote_ident(cci_name) }} ON {{ mssql__qualified_relation(relation) }} - CREATE CLUSTERED COLUMNSTORE INDEX {{ mssql__quote_ident(cci_name) }} - ON {{ mssql__qualified_relation(relation) }} + DROP INDEX {{ adapter.quote(cci_name) }} ON {{ relation }} + CREATE CLUSTERED COLUMNSTORE INDEX {{ adapter.quote(cci_name) }} + ON {{ relation }} {% endmacro %} {% macro drop_xml_indexes() -%} @@ -170,17 +154,17 @@ {% macro create_clustered_index(columns, unique=False, relation=none) -%} {%- set _relation = relation if relation is not none else this -%} {%- set cols = [columns] if columns is string else columns -%} - {%- set idx_name = mssql__index_name(_relation, 'cidx', cols, unique=unique) -%} + {%- set idx_name = sqlserver__index_name(_relation, 'cidx', cols, unique=unique) -%} {%- set quoted_cols = [] -%} {%- for col in cols -%} - {%- do quoted_cols.append(mssql__quote_ident(col)) -%} + {%- do quoted_cols.append(adapter.quote(col)) -%} {%- endfor -%} {{ log("Creating clustered index...") }} if not exists(select 1 from sys.indexes {{ information_schema_hints() }} - where name = N'{{ idx_name }}' - and object_id = OBJECT_ID(N'{{ mssql__qualified_relation(_relation) }}') + where name = N'{{ escape_single_quotes(idx_name) }}' + and object_id = OBJECT_ID(N'{{ escape_single_quotes(_relation) }}') ) begin @@ -189,8 +173,8 @@ unique {% endif %} clustered index - [{{ idx_name }}] - on {{ mssql__qualified_relation(_relation) }} ({{ quoted_cols | join(', ') }}) + {{ adapter.quote(idx_name) }} + on {{ _relation }} ({{ quoted_cols | join(', ') }}) end {%- endmacro %} @@ -200,26 +184,26 @@ {%- set _relation = relation if relation is not none else this -%} {%- set cols = [columns] if columns is string else columns -%} {%- set incs = [] if not includes else ([includes] if includes is string else includes) -%} - {%- set idx_name = mssql__index_name(_relation, 'nidx', cols, includes=incs) -%} + {%- set idx_name = sqlserver__index_name(_relation, 'nidx', cols, includes=incs) -%} {%- set quoted_cols = [] -%} {%- for col in cols -%} - {%- do quoted_cols.append(mssql__quote_ident(col)) -%} + {%- do quoted_cols.append(adapter.quote(col)) -%} {%- endfor -%} {%- set quoted_incs = [] -%} {%- for inc in incs -%} - {%- do quoted_incs.append(mssql__quote_ident(inc)) -%} + {%- do quoted_incs.append(adapter.quote(inc)) -%} {%- endfor -%} {{ log("Creating nonclustered index...") }} if not exists(select 1 from sys.indexes {{ information_schema_hints() }} - where name = N'{{ idx_name }}' - and object_id = OBJECT_ID(N'{{ mssql__qualified_relation(_relation) }}') + where name = N'{{ escape_single_quotes(idx_name) }}' + and object_id = OBJECT_ID(N'{{ escape_single_quotes(_relation) }}') ) begin create nonclustered index - [{{ idx_name }}] - on {{ mssql__qualified_relation(_relation) }} ({{ quoted_cols | join(', ') }}) + {{ adapter.quote(idx_name) }} + on {{ _relation }} ({{ quoted_cols | join(', ') }}) {% if quoted_incs -%} include ({{ quoted_incs | join(', ') }}) {% endif %} diff --git a/tests/unit/adapters/mssql/test_indexes.py b/tests/unit/adapters/mssql/test_indexes.py index 21b09298b..9b051eb93 100644 --- a/tests/unit/adapters/mssql/test_indexes.py +++ b/tests/unit/adapters/mssql/test_indexes.py @@ -4,10 +4,9 @@ Mirrors the style of test_generate_schema_name.py: inline Jinja, no DB. Coverage: - * mssql__quote_ident - bracket escaping - * mssql__qualified_relation - 3-part name assembly - * mssql__strip_dbt_suffix - __dbt_tmp / __dbt_backup stripping - * mssql__index_name - determinism, 116-char cap, hashing, + * relation rendering - qualified relation quoting + * sqlserver__strip_dbt_suffix - __dbt_tmp / __dbt_backup stripping + * sqlserver__index_name - determinism, 116-char cap, hashing, immunity to __dbt_tmp suffix * sqlserver__index_exists - emitted SQL shape (OBJECT_ID scoped) * create_clustered_index - quoted columns, IF NOT EXISTS wrapper @@ -22,6 +21,8 @@ import pytest from jinja2.runtime import Macro as _Jinja2Macro +from dbt.adapters.sqlserver.sqlserver_adapter import SQLServerAdapter + class _MacroReturn(BaseException): def __init__(self, value): @@ -86,6 +87,8 @@ def local_md5(s): "information_schema_hints": lambda: "", "log": lambda *a, **kw: "", "get_use_database_sql": lambda db: f"USE [{db}];", + "escape_single_quotes": lambda value: str(value).replace("'", "''"), + "adapter": SQLServerAdapter, "this": _FakeRelation("mydb", "myschema", "my_model"), "return": lambda v: (_ for _ in ()).throw(_MacroReturn(v)), } @@ -103,32 +106,6 @@ def _normalize_ws(s): return re.sub(r"\s+", " ", s).strip() -class TestQuoteIdent: - def test_simple_name(self): - assert _render("mssql__quote_ident('foo')") == "[foo]" - - def test_name_with_space(self): - assert _render("mssql__quote_ident('weird name')") == "[weird name]" - - def test_name_with_close_bracket_is_escaped(self): - assert _render("mssql__quote_ident('we][ird')") == "[we]][ird]" - - def test_name_with_dot_is_not_split(self): - assert _render("mssql__quote_ident('a.b')") == "[a.b]" - - -class TestQualifiedRelation: - def test_three_part_name(self): - rel = _FakeRelation("mydb", "myschema", "my_model") - out = _render("mssql__qualified_relation(rel)", rel=rel) - assert out == "[mydb].[myschema].[my_model]" - - def test_escapes_brackets_in_every_part(self): - rel = _FakeRelation("d]b", "sch]ema", "id]ent") - out = _render("mssql__qualified_relation(rel)", rel=rel) - assert out == "[d]]b].[sch]]ema].[id]]ent]" - - class TestStripDbtSuffix: @pytest.mark.parametrize( "identifier, expected", @@ -141,76 +118,76 @@ class TestStripDbtSuffix: ], ) def test_strip(self, identifier, expected): - assert _render(f"mssql__strip_dbt_suffix('{identifier}')") == expected + assert _render(f"sqlserver__strip_dbt_suffix('{identifier}')") == expected class TestIndexName: def test_deterministic(self): rel = _FakeRelation("d", "s", "t") - a = _render("mssql__index_name(rel, 'nidx', ['x','y'])", rel=rel) - b = _render("mssql__index_name(rel, 'nidx', ['x','y'])", rel=rel) + a = _render("sqlserver__index_name(rel, 'nidx', ['x','y'])", rel=rel) + b = _render("sqlserver__index_name(rel, 'nidx', ['x','y'])", rel=rel) assert a == b def test_column_order_changes_name(self): rel = _FakeRelation("d", "s", "t") - a = _render("mssql__index_name(rel, 'nidx', ['x','y'])", rel=rel) - b = _render("mssql__index_name(rel, 'nidx', ['y','x'])", rel=rel) + a = _render("sqlserver__index_name(rel, 'nidx', ['x','y'])", rel=rel) + b = _render("sqlserver__index_name(rel, 'nidx', ['y','x'])", rel=rel) assert a != b def test_unique_flag_changes_name(self): rel = _FakeRelation("d", "s", "t") - a = _render("mssql__index_name(rel, 'cidx', ['x'], unique=False)", rel=rel) - b = _render("mssql__index_name(rel, 'cidx', ['x'], unique=True)", rel=rel) + a = _render("sqlserver__index_name(rel, 'cidx', ['x'], unique=False)", rel=rel) + b = _render("sqlserver__index_name(rel, 'cidx', ['x'], unique=True)", rel=rel) assert a != b def test_includes_changes_name(self): rel = _FakeRelation("d", "s", "t") - a = _render("mssql__index_name(rel, 'nidx', ['x'])", rel=rel) - b = _render("mssql__index_name(rel, 'nidx', ['x'], includes=['y'])", rel=rel) + a = _render("sqlserver__index_name(rel, 'nidx', ['x'])", rel=rel) + b = _render("sqlserver__index_name(rel, 'nidx', ['x'], includes=['y'])", rel=rel) assert a != b def test_accepts_string_columns(self): rel = _FakeRelation("d", "s", "t") - name = _render("mssql__index_name(rel, 'nidx', 'x')", rel=rel) + name = _render("sqlserver__index_name(rel, 'nidx', 'x')", rel=rel) assert name.startswith("nidx_t_") def test_false_includes_are_treated_as_empty(self): rel = _FakeRelation("d", "s", "t") - a = _render("mssql__index_name(rel, 'nidx', ['x'], includes=false)", rel=rel) - b = _render("mssql__index_name(rel, 'nidx', ['x'])", rel=rel) + a = _render("sqlserver__index_name(rel, 'nidx', ['x'], includes=false)", rel=rel) + b = _render("sqlserver__index_name(rel, 'nidx', ['x'])", rel=rel) assert a == b def test_dbt_tmp_suffix_does_not_affect_name(self): a = _render( - "mssql__index_name(rel, 'cci', ['__all__'])", + "sqlserver__index_name(rel, 'cci', ['__all__'])", rel=_FakeRelation("d", "s", "my_model"), ) b = _render( - "mssql__index_name(rel, 'cci', ['__all__'])", + "sqlserver__index_name(rel, 'cci', ['__all__'])", rel=_FakeRelation("d", "s", "my_model__dbt_tmp"), ) c = _render( - "mssql__index_name(rel, 'cci', ['__all__'])", + "sqlserver__index_name(rel, 'cci', ['__all__'])", rel=_FakeRelation("d", "s", "my_model__dbt_backup"), ) assert a == b == c def test_length_capped_at_116_chars(self): rel = _FakeRelation("d", "s", "x" * 500) - name = _render("mssql__index_name(rel, 'nidx', ['c'])", rel=rel) + name = _render("sqlserver__index_name(rel, 'nidx', ['c'])", rel=rel) assert len(name) <= 116 def test_long_name_still_unique_per_signature(self): rel = _FakeRelation("d", "s", "x" * 500) - a = _render("mssql__index_name(rel, 'nidx', ['c1'])", rel=rel) - b = _render("mssql__index_name(rel, 'nidx', ['c2'])", rel=rel) + a = _render("sqlserver__index_name(rel, 'nidx', ['c1'])", rel=rel) + b = _render("sqlserver__index_name(rel, 'nidx', ['c2'])", rel=rel) assert a != b assert len(a) <= 116 assert len(b) <= 116 def test_brackets_stripped_from_readable_part(self): rel = _FakeRelation("d", "s", "[weird]") - name = _render("mssql__index_name(rel, 'nidx', ['c'])", rel=rel) + name = _render("sqlserver__index_name(rel, 'nidx', ['c'])", rel=rel) assert "[" not in name and "]" not in name @@ -230,7 +207,7 @@ def test_quotes_columns(self): rel = _FakeRelation("d", "s", "t") sql = _render("create_clustered_index(['id_col', 'data'], relation=rel)", rel=rel) sql = _normalize_ws(sql) - assert "([id_col], [data])" in sql + assert '("id_col", "data")' in sql def test_wrapped_in_if_not_exists(self): rel = _FakeRelation("d", "s", "t") @@ -249,7 +226,7 @@ def test_unique_flag_emits_unique_keyword(self): def test_accepts_string_column(self): rel = _FakeRelation("d", "s", "t") sql = _normalize_ws(_render("create_clustered_index('id_col', relation=rel)", rel=rel)) - assert "([id_col])" in sql + assert '("id_col")' in sql class TestCreateNonclusteredIndex: @@ -261,14 +238,14 @@ def test_emits_nonclustered(self): def test_accepts_string_column(self): rel = _FakeRelation("d", "s", "t") sql = _normalize_ws(_render("create_nonclustered_index('c', relation=rel)", rel=rel)) - assert "([c])" in sql + assert '("c")' in sql def test_accepts_string_include_column(self): rel = _FakeRelation("d", "s", "t") sql = _normalize_ws( _render("create_nonclustered_index(['c'], includes='inc1', relation=rel)", rel=rel) ) - assert "include ([inc1])" in sql + assert 'include ("inc1")' in sql def test_include_columns_quoted(self): rel = _FakeRelation("d", "s", "t") @@ -278,7 +255,7 @@ def test_include_columns_quoted(self): rel=rel, ) ) - assert "include ([inc1], [inc2])" in sql + assert 'include ("inc1", "inc2")' in sql def test_no_include_block_when_no_includes(self): rel = _FakeRelation("d", "s", "t") @@ -292,12 +269,12 @@ def test_false_include_is_treated_as_empty(self): ) assert "include" not in sql - def test_columns_with_brackets_are_escaped(self): + def test_columns_with_brackets_are_quoted(self): rel = _FakeRelation("d", "s", "t") sql = _normalize_ws( _render("create_nonclustered_index(['we]ird'], relation=rel)", rel=rel) ) - assert "[we]]ird]" in sql + assert '"we]ird"' in sql def test_idempotent_wrapper(self): rel = _FakeRelation("d", "s", "t") @@ -318,7 +295,7 @@ def test_intermediate_relation_uses_target_name(self): target=target, ) name_re = re.compile( - r"CREATE\s+CLUSTERED\s+COLUMNSTORE\s+INDEX\s+(\[[^\]]+\])", re.IGNORECASE + r'CREATE\s+CLUSTERED\s+COLUMNSTORE\s+INDEX\s+("[^"]+"|\[[^\]]+\])', re.IGNORECASE ) name_int = name_re.search(_normalize_ws(sql_int)).group(1) name_final = name_re.search(_normalize_ws(sql_final)).group(1)