chore(catalog): dispose in-memory catalog engines and clear Python 3.13 deprecation warnings

pyproject.toml

@@ -171,8 +171,6 @@ filterwarnings = [
   "error",
   # Ignore Python version deprecation warning from google.api_core while we still support 3.10
   "ignore:You are using a Python version.*which Google will stop supporting:FutureWarning:google.api_core",
-  # Python 3.13 sqlite3 module ResourceWarnings for unclosed database connections
-  "ignore:unclosed database in <sqlite3.Connection object*:ResourceWarning",
   # Ignore Ray subprocess cleanup warnings
   "ignore:unclosed file:ResourceWarning",
   "ignore:subprocess.*is still running:ResourceWarning",

tests/catalog/test_base.py

@@ -37,7 +37,9 @@ def catalog(tmp_path: PosixPath) -> Generator[Catalog, None, None]:
 
 
 def test_load_catalog_in_memory() -> None:
-    assert load_catalog("catalog", type="in-memory")
+    catalog = load_catalog("catalog", type="in-memory")
+    assert catalog
+    catalog.close()
 
 
 def test_load_catalog_impl_not_full_path() -> None:

tests/catalog/test_sql.py

@@ -56,6 +56,7 @@ def catalog_memory(catalog_name: str, warehouse: Path) -> Generator[SqlCatalog,
     catalog.create_tables()
     yield catalog
     catalog.destroy_tables()
+    catalog.close()
 
 
 @pytest.fixture(scope="module")
@@ -68,6 +69,7 @@ def catalog_sqlite(catalog_name: str, warehouse: Path) -> Generator[SqlCatalog,
     catalog.create_tables()
     yield catalog
     catalog.destroy_tables()
+    catalog.close()
 
 
 @pytest.fixture(scope="module")
@@ -76,8 +78,10 @@ def catalog_uri(warehouse: Path) -> str:
 
 
 @pytest.fixture(scope="module")
-def alchemy_engine(catalog_uri: str) -> Engine:
-    return create_engine(catalog_uri)
+def alchemy_engine(catalog_uri: str) -> Generator[Engine, None, None]:
+    engine = create_engine(catalog_uri)
+    yield engine
+    engine.dispose()
 
 
 def test_creation_with_no_uri(catalog_name: str) -> None:
@@ -107,6 +111,7 @@ def test_creation_with_echo_parameter(catalog_name: str, warehouse: Path) -> Non
             f"Assertion failed: expected echo value {expected_echo_value}, "
             f"but got {catalog.engine._echo}. For echo_param={echo_param}"
         )
+        catalog.close()
 
 
 def test_creation_with_pool_pre_ping_parameter(catalog_name: str, warehouse: Path) -> None:
@@ -131,20 +136,20 @@ def test_creation_with_pool_pre_ping_parameter(catalog_name: str, warehouse: Pat
             f"Assertion failed: expected pool_pre_ping value {expected_pool_pre_ping_value}, "
             f"but got {catalog.engine.pool._pre_ping}. For pool_pre_ping_param={pool_pre_ping_param}"
         )
+        catalog.close()
 
 
 def test_creation_from_impl(catalog_name: str, warehouse: Path) -> None:
-    assert isinstance(
-        load_catalog(
-            catalog_name,
-            **{
-                "py-catalog-impl": "pyiceberg.catalog.sql.SqlCatalog",
-                "uri": f"sqlite:////{warehouse}/sql-catalog",
-                "warehouse": f"file://{warehouse}",
-            },
-        ),
-        SqlCatalog,
+    catalog = load_catalog(
+        catalog_name,
+        **{
+            "py-catalog-impl": "pyiceberg.catalog.sql.SqlCatalog",
+            "uri": f"sqlite:////{warehouse}/sql-catalog",
+            "warehouse": f"file://{warehouse}",
+        },
     )
+    assert isinstance(catalog, SqlCatalog)
+    catalog.close()
 
 
 def confirm_no_tables_exist(alchemy_engine: Engine) -> None:
@@ -183,6 +188,7 @@ def test_creation_when_no_tables_exist(alchemy_engine: Engine, catalog_name: str
     confirm_no_tables_exist(alchemy_engine)
     catalog = load_catalog_for_catalog_table_creation(catalog_name=catalog_name, catalog_uri=catalog_uri)
     confirm_all_tables_exist(catalog)
+    catalog.close()
 
 
 def test_creation_when_one_tables_exists(alchemy_engine: Engine, catalog_name: str, catalog_uri: str) -> None:
@@ -195,6 +201,7 @@ def test_creation_when_one_tables_exists(alchemy_engine: Engine, catalog_name: s
 
     catalog = load_catalog_for_catalog_table_creation(catalog_name=catalog_name, catalog_uri=catalog_uri)
     confirm_all_tables_exist(catalog)
+    catalog.close()
 
 
 def test_creation_when_all_tables_exists(alchemy_engine: Engine, catalog_name: str, catalog_uri: str) -> None:
@@ -208,6 +215,7 @@ def test_creation_when_all_tables_exists(alchemy_engine: Engine, catalog_name: s
 
     catalog = load_catalog_for_catalog_table_creation(catalog_name=catalog_name, catalog_uri=catalog_uri)
     confirm_all_tables_exist(catalog)
+    catalog.close()
 
 
 class TestSqlCatalogClose:

tests/cli/test_console.py

@@ -17,6 +17,7 @@
 import datetime
 import os
 import uuid
+from collections.abc import Generator
 from pathlib import PosixPath
 from unittest import mock
 from unittest.mock import MagicMock
@@ -97,12 +98,13 @@ def env_vars(mocker: MockFixture) -> None:
 
 
 @pytest.fixture(name="catalog")
-def fixture_catalog(mocker: MockFixture, tmp_path: PosixPath) -> InMemoryCatalog:
+def fixture_catalog(mocker: MockFixture, tmp_path: PosixPath) -> Generator[InMemoryCatalog, None, None]:
     in_memory_catalog = InMemoryCatalog(
         "test.in_memory.catalog", **{WAREHOUSE: tmp_path.absolute().as_posix(), "test.key": "test.value"}
     )
     mocker.patch("pyiceberg.cli.console.load_catalog", return_value=in_memory_catalog)
-    return in_memory_catalog
+    yield in_memory_catalog
+    in_memory_catalog.close()
 
 
 @pytest.fixture(name="namespace_properties")

tests/conftest.py

@@ -3149,6 +3149,7 @@ def catalog(request: pytest.FixtureRequest, tmp_path: Path) -> Generator[Catalog
     yield cat
     if hasattr(cat, "destroy_tables"):
         cat.destroy_tables()
+    cat.close()
 
 
 @pytest.fixture(params=list(_CATALOG_FACTORIES.keys()))
@@ -3161,6 +3162,7 @@ def catalog_with_warehouse(
     yield cat
     if hasattr(cat, "destroy_tables"):
         cat.destroy_tables()
+    cat.close()
 
 
 @pytest.fixture(name="random_table_identifier")

tests/integration/test_catalog.py

@@ -66,6 +66,7 @@ def memory_catalog(tmp_path: PosixPath) -> Generator[Catalog, None, None]:
     yield test_catalog
 
     clean_up(test_catalog)
+    test_catalog.close()
 
 
 @pytest.fixture(scope="function")
@@ -75,6 +76,7 @@ def sqlite_catalog_memory(warehouse: Path) -> Generator[Catalog, None, None]:
     yield test_catalog
 
     clean_up(test_catalog)
+    test_catalog.close()
 
 
 @pytest.fixture(scope="function")
@@ -84,6 +86,7 @@ def sqlite_catalog_file(warehouse: Path) -> Generator[Catalog, None, None]:
     yield test_catalog
 
     clean_up(test_catalog)
+    test_catalog.close()
 
 
 @pytest.fixture(scope="function")

tests/integration/test_writes/test_writes.py

@@ -921,6 +921,7 @@ def test_duckdb_url_import(warehouse: Path, arrow_table_with_null: pa.Table) ->
             b"\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11",
         ),
     ]
+    catalog.close()
 
 
 @pytest.mark.integration
@@ -2385,6 +2386,8 @@ def test_nanosecond_support_on_catalog(
             session_catalog, identifier, {"format-version": "2"}, schema=arrow_table_schema_with_all_timestamp_precisions
         )
 
+    catalog.close()
+
 
 @pytest.mark.integration
 @pytest.mark.parametrize("format_version", [1, 2])

tests/io/test_pyarrow.py

@@ -20,7 +20,7 @@
 import tempfile
 import uuid
 import warnings
-from collections.abc import Iterator
+from collections.abc import Generator, Iterator
 from datetime import date, datetime, timezone
 from pathlib import Path
 from typing import Any
@@ -1506,8 +1506,10 @@ def test_identity_transform_columns_projection(tmp_path: str, catalog: InMemoryC
 
 
 @pytest.fixture
-def catalog() -> InMemoryCatalog:
-    return InMemoryCatalog("test.in_memory.catalog", **{"test.key": "test.value"})
+def catalog() -> Generator[InMemoryCatalog, None, None]:
+    cat = InMemoryCatalog("test.in_memory.catalog", **{"test.key": "test.value"})
+    yield cat
+    cat.close()
 
 
 def test_projection_filter(schema_int: Schema, file_int: str) -> None:

tests/table/test_datafusion.py

@@ -16,6 +16,7 @@
 # under the License.
 
 
+from collections.abc import Generator
 from pathlib import Path
 
 import pyarrow as pa
@@ -31,13 +32,15 @@ def warehouse(tmp_path_factory: pytest.TempPathFactory) -> Path:
 
 
 @pytest.fixture(scope="session")
-def catalog(warehouse: Path) -> Catalog:
+def catalog(warehouse: Path) -> Generator[Catalog, None, None]:
     catalog = load_catalog(
         "default",
         uri=f"sqlite:///{warehouse}/pyiceberg_catalog.db",
         warehouse=f"file://{warehouse}",
     )
-    return catalog
+    yield catalog
+
+    catalog.close()
 
 
 def test_datafusion_register_pyiceberg_table(catalog: Catalog, arrow_table_with_null: pa.Table) -> None:

tests/table/test_inspect.py

@@ -15,6 +15,7 @@
 # specific language governing permissions and limitations
 # under the License.
 
+from collections.abc import Generator
 from pathlib import PosixPath
 
 import pyarrow as pa
@@ -36,10 +37,11 @@ def test_readable_bound_without_bound() -> None:
 
 
 @pytest.fixture
-def catalog(tmp_path: PosixPath) -> InMemoryCatalog:
+def catalog(tmp_path: PosixPath) -> Generator[InMemoryCatalog, None, None]:
     cat = InMemoryCatalog("test.in_memory.catalog", warehouse=tmp_path.absolute().as_posix())
     cat.create_namespace("default")
-    return cat
+    yield cat
+    cat.close()
 
 
 def test_inspect_entries_and_files_render_empty_string_bound(catalog: InMemoryCatalog) -> None:

tests/table/test_upsert.py

@@ -14,6 +14,7 @@
 # KIND, either express or implied.  See the License for the
 # specific language governing permissions and limitations
 # under the License.
+from collections.abc import Generator
 from pathlib import PosixPath
 
 import pyarrow as pa
@@ -35,10 +36,11 @@
 
 
 @pytest.fixture
-def catalog(tmp_path: PosixPath) -> InMemoryCatalog:
+def catalog(tmp_path: PosixPath) -> Generator[InMemoryCatalog, None, None]:
     catalog = InMemoryCatalog("test.in_memory.catalog", warehouse=tmp_path.absolute().as_posix())
     catalog.create_namespace("default")
-    return catalog
+    yield catalog
+    catalog.close()
 
 
 def _drop_table(catalog: Catalog, identifier: str) -> None: