From 100a0db0f2495db96bad553649a3c962d77a29bb Mon Sep 17 00:00:00 2001 From: Abanoub Doss Date: Tue, 23 Jun 2026 18:05:37 -0500 Subject: [PATCH 1/5] Fix sqlite ResourceWarning on Python 3.13 by disposing SqlCatalog engines in tests The 'unclosed database in ' ResourceWarning under Python 3.13 was suppressed via a pytest filterwarnings ignore (#2863). Root cause: test fixtures and tests created SqlCatalog instances (each opening a SQLAlchemy engine / sqlite connection pool) without ever calling the existing SqlCatalog.close() (added in #2390), so the pooled sqlite connections leaked and emitted ResourceWarning during GC. Wire close()/engine.dispose() into the relevant fixtures and tests and remove the now-unnecessary sqlite ResourceWarning filter. The ray and google (api_core / crc32c) filters remain, as those originate in those dependencies, not pyiceberg. --- pyproject.toml | 2 -- tests/catalog/test_sql.py | 32 ++++++++++++++++++++------------ tests/conftest.py | 2 ++ 3 files changed, 22 insertions(+), 14 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 397fdf55bc..1ccfa90e62 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -171,8 +171,6 @@ filterwarnings = [ "error", # Ignore Python version deprecation warning from google.api_core while we still support 3.10 "ignore:You are using a Python version.*which Google will stop supporting:FutureWarning:google.api_core", - # Python 3.13 sqlite3 module ResourceWarnings for unclosed database connections - "ignore:unclosed database in Generator[SqlCatalog, catalog.create_tables() yield catalog catalog.destroy_tables() + catalog.close() @pytest.fixture(scope="module") @@ -68,6 +69,7 @@ def catalog_sqlite(catalog_name: str, warehouse: Path) -> Generator[SqlCatalog, catalog.create_tables() yield catalog catalog.destroy_tables() + catalog.close() @pytest.fixture(scope="module") @@ -76,8 +78,10 @@ def catalog_uri(warehouse: Path) -> str: @pytest.fixture(scope="module") -def alchemy_engine(catalog_uri: str) -> Engine: - return create_engine(catalog_uri) +def alchemy_engine(catalog_uri: str) -> Generator[Engine, None, None]: + engine = create_engine(catalog_uri) + yield engine + engine.dispose() def test_creation_with_no_uri(catalog_name: str) -> None: @@ -107,6 +111,7 @@ def test_creation_with_echo_parameter(catalog_name: str, warehouse: Path) -> Non f"Assertion failed: expected echo value {expected_echo_value}, " f"but got {catalog.engine._echo}. For echo_param={echo_param}" ) + catalog.close() def test_creation_with_pool_pre_ping_parameter(catalog_name: str, warehouse: Path) -> None: @@ -131,20 +136,20 @@ def test_creation_with_pool_pre_ping_parameter(catalog_name: str, warehouse: Pat f"Assertion failed: expected pool_pre_ping value {expected_pool_pre_ping_value}, " f"but got {catalog.engine.pool._pre_ping}. For pool_pre_ping_param={pool_pre_ping_param}" ) + catalog.close() def test_creation_from_impl(catalog_name: str, warehouse: Path) -> None: - assert isinstance( - load_catalog( - catalog_name, - **{ - "py-catalog-impl": "pyiceberg.catalog.sql.SqlCatalog", - "uri": f"sqlite:////{warehouse}/sql-catalog", - "warehouse": f"file://{warehouse}", - }, - ), - SqlCatalog, + catalog = load_catalog( + catalog_name, + **{ + "py-catalog-impl": "pyiceberg.catalog.sql.SqlCatalog", + "uri": f"sqlite:////{warehouse}/sql-catalog", + "warehouse": f"file://{warehouse}", + }, ) + assert isinstance(catalog, SqlCatalog) + catalog.close() def confirm_no_tables_exist(alchemy_engine: Engine) -> None: @@ -183,6 +188,7 @@ def test_creation_when_no_tables_exist(alchemy_engine: Engine, catalog_name: str confirm_no_tables_exist(alchemy_engine) catalog = load_catalog_for_catalog_table_creation(catalog_name=catalog_name, catalog_uri=catalog_uri) confirm_all_tables_exist(catalog) + catalog.close() def test_creation_when_one_tables_exists(alchemy_engine: Engine, catalog_name: str, catalog_uri: str) -> None: @@ -195,6 +201,7 @@ def test_creation_when_one_tables_exists(alchemy_engine: Engine, catalog_name: s catalog = load_catalog_for_catalog_table_creation(catalog_name=catalog_name, catalog_uri=catalog_uri) confirm_all_tables_exist(catalog) + catalog.close() def test_creation_when_all_tables_exists(alchemy_engine: Engine, catalog_name: str, catalog_uri: str) -> None: @@ -208,6 +215,7 @@ def test_creation_when_all_tables_exists(alchemy_engine: Engine, catalog_name: s catalog = load_catalog_for_catalog_table_creation(catalog_name=catalog_name, catalog_uri=catalog_uri) confirm_all_tables_exist(catalog) + catalog.close() class TestSqlCatalogClose: diff --git a/tests/conftest.py b/tests/conftest.py index d7cdba012b..31e047072d 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -3149,6 +3149,7 @@ def catalog(request: pytest.FixtureRequest, tmp_path: Path) -> Generator[Catalog yield cat if hasattr(cat, "destroy_tables"): cat.destroy_tables() + cat.close() @pytest.fixture(params=list(_CATALOG_FACTORIES.keys())) @@ -3161,6 +3162,7 @@ def catalog_with_warehouse( yield cat if hasattr(cat, "destroy_tables"): cat.destroy_tables() + cat.close() @pytest.fixture(name="random_table_identifier") From c4e1bba35e98b45727a2c3f56d95d7ed6dab303b Mon Sep 17 00:00:00 2001 From: Abanoub Doss Date: Tue, 23 Jun 2026 18:22:49 -0500 Subject: [PATCH 2/5] Dispose InMemoryCatalog engines in upsert/inspect/base tests InMemoryCatalog is a SqlCatalog backed by sqlite:///:memory:, so its local test fixtures and load_catalog(type=in-memory) usages leaked sqlite connections (ResourceWarning under Python 3.13). Convert the local 'catalog' fixtures to yield+close and close the catalog created by test_load_catalog_in_memory. --- tests/catalog/test_base.py | 4 +++- tests/table/test_inspect.py | 6 ++++-- tests/table/test_upsert.py | 6 ++++-- 3 files changed, 11 insertions(+), 5 deletions(-) diff --git a/tests/catalog/test_base.py b/tests/catalog/test_base.py index 1a47478313..6e3a88a877 100644 --- a/tests/catalog/test_base.py +++ b/tests/catalog/test_base.py @@ -37,7 +37,9 @@ def catalog(tmp_path: PosixPath) -> Generator[Catalog, None, None]: def test_load_catalog_in_memory() -> None: - assert load_catalog("catalog", type="in-memory") + catalog = load_catalog("catalog", type="in-memory") + assert catalog + catalog.close() def test_load_catalog_impl_not_full_path() -> None: diff --git a/tests/table/test_inspect.py b/tests/table/test_inspect.py index c325af2033..524312e22e 100644 --- a/tests/table/test_inspect.py +++ b/tests/table/test_inspect.py @@ -15,6 +15,7 @@ # specific language governing permissions and limitations # under the License. +from collections.abc import Generator from pathlib import PosixPath import pyarrow as pa @@ -36,10 +37,11 @@ def test_readable_bound_without_bound() -> None: @pytest.fixture -def catalog(tmp_path: PosixPath) -> InMemoryCatalog: +def catalog(tmp_path: PosixPath) -> Generator[InMemoryCatalog, None, None]: cat = InMemoryCatalog("test.in_memory.catalog", warehouse=tmp_path.absolute().as_posix()) cat.create_namespace("default") - return cat + yield cat + cat.close() def test_inspect_entries_and_files_render_empty_string_bound(catalog: InMemoryCatalog) -> None: diff --git a/tests/table/test_upsert.py b/tests/table/test_upsert.py index 08f90c6600..6c7dabc2e5 100644 --- a/tests/table/test_upsert.py +++ b/tests/table/test_upsert.py @@ -14,6 +14,7 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. +from collections.abc import Generator from pathlib import PosixPath import pyarrow as pa @@ -35,10 +36,11 @@ @pytest.fixture -def catalog(tmp_path: PosixPath) -> InMemoryCatalog: +def catalog(tmp_path: PosixPath) -> Generator[InMemoryCatalog, None, None]: catalog = InMemoryCatalog("test.in_memory.catalog", warehouse=tmp_path.absolute().as_posix()) catalog.create_namespace("default") - return catalog + yield catalog + catalog.close() def _drop_table(catalog: Catalog, identifier: str) -> None: From 08067c74a0c0739e5ab7f3157e596720dfd27a76 Mon Sep 17 00:00:00 2001 From: Abanoub Doss Date: Tue, 23 Jun 2026 20:18:36 -0500 Subject: [PATCH 3/5] Close leaked sqlite catalogs in datafusion and integration tests --- tests/integration/test_catalog.py | 2 ++ tests/integration/test_writes/test_writes.py | 1 + tests/table/test_datafusion.py | 7 +++++-- 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/tests/integration/test_catalog.py b/tests/integration/test_catalog.py index 630cae4767..7205b4f40f 100644 --- a/tests/integration/test_catalog.py +++ b/tests/integration/test_catalog.py @@ -75,6 +75,7 @@ def sqlite_catalog_memory(warehouse: Path) -> Generator[Catalog, None, None]: yield test_catalog clean_up(test_catalog) + test_catalog.close() @pytest.fixture(scope="function") @@ -84,6 +85,7 @@ def sqlite_catalog_file(warehouse: Path) -> Generator[Catalog, None, None]: yield test_catalog clean_up(test_catalog) + test_catalog.close() @pytest.fixture(scope="function") diff --git a/tests/integration/test_writes/test_writes.py b/tests/integration/test_writes/test_writes.py index 2a0c50a921..0e18fca909 100644 --- a/tests/integration/test_writes/test_writes.py +++ b/tests/integration/test_writes/test_writes.py @@ -921,6 +921,7 @@ def test_duckdb_url_import(warehouse: Path, arrow_table_with_null: pa.Table) -> b"\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11", ), ] + catalog.close() @pytest.mark.integration diff --git a/tests/table/test_datafusion.py b/tests/table/test_datafusion.py index 136145ce8a..0f5628ec4d 100644 --- a/tests/table/test_datafusion.py +++ b/tests/table/test_datafusion.py @@ -16,6 +16,7 @@ # under the License. +from collections.abc import Generator from pathlib import Path import pyarrow as pa @@ -31,13 +32,15 @@ def warehouse(tmp_path_factory: pytest.TempPathFactory) -> Path: @pytest.fixture(scope="session") -def catalog(warehouse: Path) -> Catalog: +def catalog(warehouse: Path) -> Generator[Catalog, None, None]: catalog = load_catalog( "default", uri=f"sqlite:///{warehouse}/pyiceberg_catalog.db", warehouse=f"file://{warehouse}", ) - return catalog + yield catalog + + catalog.close() def test_datafusion_register_pyiceberg_table(catalog: Catalog, arrow_table_with_null: pa.Table) -> None: From ba36a2cf562c471c152005de4dcfaf40e40ac90b Mon Sep 17 00:00:00 2001 From: Abanoub Doss Date: Tue, 23 Jun 2026 20:25:37 -0500 Subject: [PATCH 4/5] Close leaked InMemoryCatalog in cli and pyarrow test fixtures --- tests/cli/test_console.py | 6 ++++-- tests/io/test_pyarrow.py | 8 +++++--- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/tests/cli/test_console.py b/tests/cli/test_console.py index 27a1bfebe4..891f488d91 100644 --- a/tests/cli/test_console.py +++ b/tests/cli/test_console.py @@ -17,6 +17,7 @@ import datetime import os import uuid +from collections.abc import Generator from pathlib import PosixPath from unittest import mock from unittest.mock import MagicMock @@ -97,12 +98,13 @@ def env_vars(mocker: MockFixture) -> None: @pytest.fixture(name="catalog") -def fixture_catalog(mocker: MockFixture, tmp_path: PosixPath) -> InMemoryCatalog: +def fixture_catalog(mocker: MockFixture, tmp_path: PosixPath) -> Generator[InMemoryCatalog, None, None]: in_memory_catalog = InMemoryCatalog( "test.in_memory.catalog", **{WAREHOUSE: tmp_path.absolute().as_posix(), "test.key": "test.value"} ) mocker.patch("pyiceberg.cli.console.load_catalog", return_value=in_memory_catalog) - return in_memory_catalog + yield in_memory_catalog + in_memory_catalog.close() @pytest.fixture(name="namespace_properties") diff --git a/tests/io/test_pyarrow.py b/tests/io/test_pyarrow.py index 407ec611fd..a456f921dc 100644 --- a/tests/io/test_pyarrow.py +++ b/tests/io/test_pyarrow.py @@ -20,7 +20,7 @@ import tempfile import uuid import warnings -from collections.abc import Iterator +from collections.abc import Generator, Iterator from datetime import date, datetime, timezone from pathlib import Path from typing import Any @@ -1506,8 +1506,10 @@ def test_identity_transform_columns_projection(tmp_path: str, catalog: InMemoryC @pytest.fixture -def catalog() -> InMemoryCatalog: - return InMemoryCatalog("test.in_memory.catalog", **{"test.key": "test.value"}) +def catalog() -> Generator[InMemoryCatalog, None, None]: + cat = InMemoryCatalog("test.in_memory.catalog", **{"test.key": "test.value"}) + yield cat + cat.close() def test_projection_filter(schema_int: Schema, file_int: str) -> None: From 5ab5415ac9dff4b2846df4c3cbd103471ba35c48 Mon Sep 17 00:00:00 2001 From: Abanoub Doss Date: Tue, 23 Jun 2026 20:30:13 -0500 Subject: [PATCH 5/5] Close leaked in-memory catalogs in integration tests --- tests/integration/test_catalog.py | 1 + tests/integration/test_writes/test_writes.py | 2 ++ 2 files changed, 3 insertions(+) diff --git a/tests/integration/test_catalog.py b/tests/integration/test_catalog.py index 7205b4f40f..587090ce25 100644 --- a/tests/integration/test_catalog.py +++ b/tests/integration/test_catalog.py @@ -66,6 +66,7 @@ def memory_catalog(tmp_path: PosixPath) -> Generator[Catalog, None, None]: yield test_catalog clean_up(test_catalog) + test_catalog.close() @pytest.fixture(scope="function") diff --git a/tests/integration/test_writes/test_writes.py b/tests/integration/test_writes/test_writes.py index 0e18fca909..f79853e2be 100644 --- a/tests/integration/test_writes/test_writes.py +++ b/tests/integration/test_writes/test_writes.py @@ -2386,6 +2386,8 @@ def test_nanosecond_support_on_catalog( session_catalog, identifier, {"format-version": "2"}, schema=arrow_table_schema_with_all_timestamp_precisions ) + catalog.close() + @pytest.mark.integration @pytest.mark.parametrize("format_version", [1, 2])