From 8696a92c11b44d2e52f50661d7126d6be74733cb Mon Sep 17 00:00:00 2001 From: Yuya Ebihara Date: Sat, 9 May 2026 09:49:58 +0900 Subject: [PATCH] REST: Add pagination support for list_namespaces --- pyiceberg/catalog/rest/__init__.py | 43 +++++++++++++------ tests/catalog/test_rest.py | 69 ++++++++++++++++++++++++++++++ 2 files changed, 100 insertions(+), 12 deletions(-) diff --git a/pyiceberg/catalog/rest/__init__.py b/pyiceberg/catalog/rest/__init__.py index ca0ff75e8c..6c7335c86a 100644 --- a/pyiceberg/catalog/rest/__init__.py +++ b/pyiceberg/catalog/rest/__init__.py @@ -334,6 +334,7 @@ def _parse_endpoints(cls, v: list[str] | None) -> set[Endpoint] | None: class ListNamespaceResponse(IcebergBaseModel): namespaces: list[Identifier] = Field() + next_page_token: str | None = Field(default=None, alias="next-page-token") class NamespaceResponse(IcebergBaseModel): @@ -1182,19 +1183,37 @@ def drop_namespace(self, namespace: str | Identifier) -> None: def list_namespaces(self, namespace: str | Identifier = ()) -> list[Identifier]: self._check_endpoint(Capability.V1_LIST_NAMESPACES) namespace_tuple = self.identifier_to_tuple(namespace) - response = self._session.get( - self.url( - f"{Endpoints.list_namespaces}?parent={self._encode_namespace_path(namespace_tuple)}" - if namespace_tuple - else Endpoints.list_namespaces - ), - ) - try: - response.raise_for_status() - except HTTPError as exc: - _handle_non_200_response(exc, {404: NoSuchNamespaceError}) - return ListNamespaceResponse.model_validate_json(response.text).namespaces + all_namespaces: list[Identifier] = [] + page_token: str | None = None + + while True: + # Build URL with pagination params + if namespace_tuple: + base_url = f"{Endpoints.list_namespaces}?parent={self._encode_namespace_path(namespace_tuple)}" + separator = "&" + else: + base_url = Endpoints.list_namespaces + separator = "?" + + # Add page token if present + url = f"{base_url}{separator}pageToken={page_token}" if page_token else base_url + + response = self._session.get(self.url(url)) + try: + response.raise_for_status() + except HTTPError as exc: + _handle_non_200_response(exc, {404: NoSuchNamespaceError}) + + parsed = ListNamespaceResponse.model_validate_json(response.text) + all_namespaces.extend(parsed.namespaces) + + # Check if more pages exist + if not parsed.next_page_token: + break + page_token = parsed.next_page_token + + return all_namespaces @retry(**_RETRY_ARGS) def load_namespace_properties(self, namespace: str | Identifier) -> Properties: diff --git a/tests/catalog/test_rest.py b/tests/catalog/test_rest.py index 7977892635..94f80ff2a9 100644 --- a/tests/catalog/test_rest.py +++ b/tests/catalog/test_rest.py @@ -736,6 +736,75 @@ def test_list_namespace_with_parent_200(rest_mock: Mocker) -> None: ] +def test_list_namespaces_paginated_200(rest_mock: Mocker) -> None: + # First page with next-page-token + rest_mock.get( + f"{TEST_URI}v1/namespaces", + json={ + "namespaces": [["ns1"], ["ns2"]], + "next-page-token": "page2token", + }, + status_code=200, + request_headers=TEST_HEADERS, + ) + # Second page with next-page-token + rest_mock.get( + f"{TEST_URI}v1/namespaces?pageToken=page2token", + json={ + "namespaces": [["ns3"], ["ns4"]], + "next-page-token": "page3token", + }, + status_code=200, + request_headers=TEST_HEADERS, + ) + # Third page without next-page-token (last page) + rest_mock.get( + f"{TEST_URI}v1/namespaces?pageToken=page3token", + json={ + "namespaces": [["ns5"]], + }, + status_code=200, + request_headers=TEST_HEADERS, + ) + + result = RestCatalog("rest", uri=TEST_URI, token=TEST_TOKEN).list_namespaces() + assert result == [ + ("ns1",), + ("ns2",), + ("ns3",), + ("ns4",), + ("ns5",), + ] + + +def test_list_namespaces_with_parent_paginated_200(rest_mock: Mocker) -> None: + # First page + rest_mock.get( + f"{TEST_URI}v1/namespaces?parent=accounting", + json={ + "namespaces": [["accounting", "tax"]], + "next-page-token": "page2", + }, + status_code=200, + request_headers=TEST_HEADERS, + ) + # Second page (last) + rest_mock.get( + f"{TEST_URI}v1/namespaces?parent=accounting&pageToken=page2", + json={ + "namespaces": [["accounting", "payroll"]], + }, + status_code=200, + request_headers=TEST_HEADERS, + ) + + result = RestCatalog("rest", uri=TEST_URI, token=TEST_TOKEN).list_namespaces(("accounting",)) + assert result == [ + ("accounting", "tax"), + ("accounting", "payroll"), + ] + + def test_list_namespace_with_parent_404(rest_mock: Mocker) -> None: rest_mock.get( f"{TEST_URI}v1/namespaces?parent=some_namespace",