diff --git a/pyproject.toml b/pyproject.toml index 397fdf55bc..1ccfa90e62 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -171,8 +171,6 @@ filterwarnings = [ "error", # Ignore Python version deprecation warning from google.api_core while we still support 3.10 "ignore:You are using a Python version.*which Google will stop supporting:FutureWarning:google.api_core", - # Python 3.13 sqlite3 module ResourceWarnings for unclosed database connections - "ignore:unclosed database in Generator[Catalog, None, None]: def test_load_catalog_in_memory() -> None: - assert load_catalog("catalog", type="in-memory") + catalog = load_catalog("catalog", type="in-memory") + assert catalog + catalog.close() def test_load_catalog_impl_not_full_path() -> None: diff --git a/tests/catalog/test_sql.py b/tests/catalog/test_sql.py index f6846195fe..1ba0c07399 100644 --- a/tests/catalog/test_sql.py +++ b/tests/catalog/test_sql.py @@ -56,6 +56,7 @@ def catalog_memory(catalog_name: str, warehouse: Path) -> Generator[SqlCatalog, catalog.create_tables() yield catalog catalog.destroy_tables() + catalog.close() @pytest.fixture(scope="module") @@ -68,6 +69,7 @@ def catalog_sqlite(catalog_name: str, warehouse: Path) -> Generator[SqlCatalog, catalog.create_tables() yield catalog catalog.destroy_tables() + catalog.close() @pytest.fixture(scope="module") @@ -76,8 +78,10 @@ def catalog_uri(warehouse: Path) -> str: @pytest.fixture(scope="module") -def alchemy_engine(catalog_uri: str) -> Engine: - return create_engine(catalog_uri) +def alchemy_engine(catalog_uri: str) -> Generator[Engine, None, None]: + engine = create_engine(catalog_uri) + yield engine + engine.dispose() def test_creation_with_no_uri(catalog_name: str) -> None: @@ -107,6 +111,7 @@ def test_creation_with_echo_parameter(catalog_name: str, warehouse: Path) -> Non f"Assertion failed: expected echo value {expected_echo_value}, " f"but got {catalog.engine._echo}. For echo_param={echo_param}" ) + catalog.close() def test_creation_with_pool_pre_ping_parameter(catalog_name: str, warehouse: Path) -> None: @@ -131,20 +136,20 @@ def test_creation_with_pool_pre_ping_parameter(catalog_name: str, warehouse: Pat f"Assertion failed: expected pool_pre_ping value {expected_pool_pre_ping_value}, " f"but got {catalog.engine.pool._pre_ping}. For pool_pre_ping_param={pool_pre_ping_param}" ) + catalog.close() def test_creation_from_impl(catalog_name: str, warehouse: Path) -> None: - assert isinstance( - load_catalog( - catalog_name, - **{ - "py-catalog-impl": "pyiceberg.catalog.sql.SqlCatalog", - "uri": f"sqlite:////{warehouse}/sql-catalog", - "warehouse": f"file://{warehouse}", - }, - ), - SqlCatalog, + catalog = load_catalog( + catalog_name, + **{ + "py-catalog-impl": "pyiceberg.catalog.sql.SqlCatalog", + "uri": f"sqlite:////{warehouse}/sql-catalog", + "warehouse": f"file://{warehouse}", + }, ) + assert isinstance(catalog, SqlCatalog) + catalog.close() def confirm_no_tables_exist(alchemy_engine: Engine) -> None: @@ -183,6 +188,7 @@ def test_creation_when_no_tables_exist(alchemy_engine: Engine, catalog_name: str confirm_no_tables_exist(alchemy_engine) catalog = load_catalog_for_catalog_table_creation(catalog_name=catalog_name, catalog_uri=catalog_uri) confirm_all_tables_exist(catalog) + catalog.close() def test_creation_when_one_tables_exists(alchemy_engine: Engine, catalog_name: str, catalog_uri: str) -> None: @@ -195,6 +201,7 @@ def test_creation_when_one_tables_exists(alchemy_engine: Engine, catalog_name: s catalog = load_catalog_for_catalog_table_creation(catalog_name=catalog_name, catalog_uri=catalog_uri) confirm_all_tables_exist(catalog) + catalog.close() def test_creation_when_all_tables_exists(alchemy_engine: Engine, catalog_name: str, catalog_uri: str) -> None: @@ -208,6 +215,7 @@ def test_creation_when_all_tables_exists(alchemy_engine: Engine, catalog_name: s catalog = load_catalog_for_catalog_table_creation(catalog_name=catalog_name, catalog_uri=catalog_uri) confirm_all_tables_exist(catalog) + catalog.close() class TestSqlCatalogClose: diff --git a/tests/cli/test_console.py b/tests/cli/test_console.py index 27a1bfebe4..891f488d91 100644 --- a/tests/cli/test_console.py +++ b/tests/cli/test_console.py @@ -17,6 +17,7 @@ import datetime import os import uuid +from collections.abc import Generator from pathlib import PosixPath from unittest import mock from unittest.mock import MagicMock @@ -97,12 +98,13 @@ def env_vars(mocker: MockFixture) -> None: @pytest.fixture(name="catalog") -def fixture_catalog(mocker: MockFixture, tmp_path: PosixPath) -> InMemoryCatalog: +def fixture_catalog(mocker: MockFixture, tmp_path: PosixPath) -> Generator[InMemoryCatalog, None, None]: in_memory_catalog = InMemoryCatalog( "test.in_memory.catalog", **{WAREHOUSE: tmp_path.absolute().as_posix(), "test.key": "test.value"} ) mocker.patch("pyiceberg.cli.console.load_catalog", return_value=in_memory_catalog) - return in_memory_catalog + yield in_memory_catalog + in_memory_catalog.close() @pytest.fixture(name="namespace_properties") diff --git a/tests/conftest.py b/tests/conftest.py index d7cdba012b..31e047072d 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -3149,6 +3149,7 @@ def catalog(request: pytest.FixtureRequest, tmp_path: Path) -> Generator[Catalog yield cat if hasattr(cat, "destroy_tables"): cat.destroy_tables() + cat.close() @pytest.fixture(params=list(_CATALOG_FACTORIES.keys())) @@ -3161,6 +3162,7 @@ def catalog_with_warehouse( yield cat if hasattr(cat, "destroy_tables"): cat.destroy_tables() + cat.close() @pytest.fixture(name="random_table_identifier") diff --git a/tests/integration/test_catalog.py b/tests/integration/test_catalog.py index 630cae4767..587090ce25 100644 --- a/tests/integration/test_catalog.py +++ b/tests/integration/test_catalog.py @@ -66,6 +66,7 @@ def memory_catalog(tmp_path: PosixPath) -> Generator[Catalog, None, None]: yield test_catalog clean_up(test_catalog) + test_catalog.close() @pytest.fixture(scope="function") @@ -75,6 +76,7 @@ def sqlite_catalog_memory(warehouse: Path) -> Generator[Catalog, None, None]: yield test_catalog clean_up(test_catalog) + test_catalog.close() @pytest.fixture(scope="function") @@ -84,6 +86,7 @@ def sqlite_catalog_file(warehouse: Path) -> Generator[Catalog, None, None]: yield test_catalog clean_up(test_catalog) + test_catalog.close() @pytest.fixture(scope="function") diff --git a/tests/integration/test_writes/test_writes.py b/tests/integration/test_writes/test_writes.py index 2a0c50a921..f79853e2be 100644 --- a/tests/integration/test_writes/test_writes.py +++ b/tests/integration/test_writes/test_writes.py @@ -921,6 +921,7 @@ def test_duckdb_url_import(warehouse: Path, arrow_table_with_null: pa.Table) -> b"\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11", ), ] + catalog.close() @pytest.mark.integration @@ -2385,6 +2386,8 @@ def test_nanosecond_support_on_catalog( session_catalog, identifier, {"format-version": "2"}, schema=arrow_table_schema_with_all_timestamp_precisions ) + catalog.close() + @pytest.mark.integration @pytest.mark.parametrize("format_version", [1, 2]) diff --git a/tests/io/test_pyarrow.py b/tests/io/test_pyarrow.py index 407ec611fd..a456f921dc 100644 --- a/tests/io/test_pyarrow.py +++ b/tests/io/test_pyarrow.py @@ -20,7 +20,7 @@ import tempfile import uuid import warnings -from collections.abc import Iterator +from collections.abc import Generator, Iterator from datetime import date, datetime, timezone from pathlib import Path from typing import Any @@ -1506,8 +1506,10 @@ def test_identity_transform_columns_projection(tmp_path: str, catalog: InMemoryC @pytest.fixture -def catalog() -> InMemoryCatalog: - return InMemoryCatalog("test.in_memory.catalog", **{"test.key": "test.value"}) +def catalog() -> Generator[InMemoryCatalog, None, None]: + cat = InMemoryCatalog("test.in_memory.catalog", **{"test.key": "test.value"}) + yield cat + cat.close() def test_projection_filter(schema_int: Schema, file_int: str) -> None: diff --git a/tests/table/test_datafusion.py b/tests/table/test_datafusion.py index 136145ce8a..0f5628ec4d 100644 --- a/tests/table/test_datafusion.py +++ b/tests/table/test_datafusion.py @@ -16,6 +16,7 @@ # under the License. +from collections.abc import Generator from pathlib import Path import pyarrow as pa @@ -31,13 +32,15 @@ def warehouse(tmp_path_factory: pytest.TempPathFactory) -> Path: @pytest.fixture(scope="session") -def catalog(warehouse: Path) -> Catalog: +def catalog(warehouse: Path) -> Generator[Catalog, None, None]: catalog = load_catalog( "default", uri=f"sqlite:///{warehouse}/pyiceberg_catalog.db", warehouse=f"file://{warehouse}", ) - return catalog + yield catalog + + catalog.close() def test_datafusion_register_pyiceberg_table(catalog: Catalog, arrow_table_with_null: pa.Table) -> None: diff --git a/tests/table/test_inspect.py b/tests/table/test_inspect.py index c325af2033..524312e22e 100644 --- a/tests/table/test_inspect.py +++ b/tests/table/test_inspect.py @@ -15,6 +15,7 @@ # specific language governing permissions and limitations # under the License. +from collections.abc import Generator from pathlib import PosixPath import pyarrow as pa @@ -36,10 +37,11 @@ def test_readable_bound_without_bound() -> None: @pytest.fixture -def catalog(tmp_path: PosixPath) -> InMemoryCatalog: +def catalog(tmp_path: PosixPath) -> Generator[InMemoryCatalog, None, None]: cat = InMemoryCatalog("test.in_memory.catalog", warehouse=tmp_path.absolute().as_posix()) cat.create_namespace("default") - return cat + yield cat + cat.close() def test_inspect_entries_and_files_render_empty_string_bound(catalog: InMemoryCatalog) -> None: diff --git a/tests/table/test_upsert.py b/tests/table/test_upsert.py index 08f90c6600..6c7dabc2e5 100644 --- a/tests/table/test_upsert.py +++ b/tests/table/test_upsert.py @@ -14,6 +14,7 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. +from collections.abc import Generator from pathlib import PosixPath import pyarrow as pa @@ -35,10 +36,11 @@ @pytest.fixture -def catalog(tmp_path: PosixPath) -> InMemoryCatalog: +def catalog(tmp_path: PosixPath) -> Generator[InMemoryCatalog, None, None]: catalog = InMemoryCatalog("test.in_memory.catalog", warehouse=tmp_path.absolute().as_posix()) catalog.create_namespace("default") - return catalog + yield catalog + catalog.close() def _drop_table(catalog: Catalog, identifier: str) -> None: