From bd5af0fe38a332de639318730c25ec0b05be026f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9s=20Contreras=20Guill=C3=A9n?= Date: Wed, 10 Jun 2026 02:06:45 +0200 Subject: [PATCH 1/6] feat(config-server): add GitConfigBackend adapter with GitPython + config-server-git extra MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit New `src/pyfly/config_server/adapters/git.py`: `GitConfigBackend` that lazily clones a Git repo (local path or remote URI) and composes a `FilesystemConfigBackend` over the working tree for all file-read logic. `save()` commits changes locally (no push — documented out of scope). `refresh()` pulls from origin when a remote exists. - Lazy import via `_require_git()` helper; friendly ImportError when absent - `pyproject.toml`: adds `config-server-git = ["GitPython>=3.1"]` extra and includes it in `full`; `uv lock` resolves gitdb/smmap/gitpython - `tests/config_server/test_git_backend.py`: six `importorskip("git")` tests covering fetch, list, save-commits, save-updates, refresh-no-remote, and missing-returns-None — all using a local on-disk repo (no network) --- pyproject.toml | 5 +- src/pyfly/config_server/adapters/git.py | 175 ++++++++++++++++++++++++ tests/config_server/test_git_backend.py | 169 +++++++++++++++++++++++ uv.lock | 42 +++++- 4 files changed, 388 insertions(+), 3 deletions(-) create mode 100644 src/pyfly/config_server/adapters/git.py create mode 100644 tests/config_server/test_git_backend.py diff --git a/pyproject.toml b/pyproject.toml index cb1e91ac..9a238105 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -88,6 +88,9 @@ cache = [ client = [ "httpx>=0.28.1", ] +config-server-git = [ + "GitPython>=3.1", +] grpc = [ "grpcio>=1.60.0", ] @@ -150,7 +153,7 @@ web-fastapi = [ "uvloop>=0.22.1; sys_platform != 'win32'", ] full = [ - "pyfly[web,data-relational,data-document,postgresql,eda,cache,client,grpc,websocket,ecm-aws,ecm-azure,observability,security,scheduling,cli,shell,kafka,rabbitmq,redis,granian,fastapi,hypercorn,idp-azure,idp-keycloak,idp-cognito,notifications]", + "pyfly[web,data-relational,data-document,postgresql,eda,cache,client,grpc,websocket,ecm-aws,ecm-azure,observability,security,scheduling,cli,shell,kafka,rabbitmq,redis,granian,fastapi,hypercorn,idp-azure,idp-keycloak,idp-cognito,notifications,config-server-git]", ] [dependency-groups] diff --git a/src/pyfly/config_server/adapters/git.py b/src/pyfly/config_server/adapters/git.py new file mode 100644 index 00000000..b8a7ea48 --- /dev/null +++ b/src/pyfly/config_server/adapters/git.py @@ -0,0 +1,175 @@ +# Copyright 2026 Firefly Software Foundation. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Git-backed config backend — wraps a local or remote Git repository. + +Config files are read from the working tree of a cloned (or locally accessed) +repository; writes are committed locally. Pushing to a remote is **out of +scope** — call ``refresh()`` after a remote push to pull the latest commits. + +Requires GitPython: ``pip install pyfly[config-server-git]``. +""" + +from __future__ import annotations + +import asyncio +import importlib.util +import logging +import tempfile +from typing import Any + +from pyfly.config_server.backend import ConfigSource, FilesystemConfigBackend + +_logger = logging.getLogger(__name__) + + +def _require_git() -> Any: + """Import and return the ``git`` module, raising a friendly ImportError if absent.""" + try: + import git + + return git + except ImportError as exc: + msg = "GitConfigBackend requires GitPython — `pip install pyfly[config-server-git]`" + raise ImportError(msg) from exc + + +class GitConfigBackend: + """``ConfigBackend`` backed by a Git repository. + + On first use the repository is cloned (or, for a local ``file://`` / path + URI, the on-disk repo is reused) into *clone_dir* (or a tempdir when + *clone_dir* is ``None``). The working tree is then delegated to a + :class:`~pyfly.config_server.backend.FilesystemConfigBackend` so all the + file-search and merge logic is shared. + + Parameters + ---------- + uri: + Any URI accepted by ``git clone``: ``https://``, ``git@``, or a local + path (``/path/to/repo`` or ``file:///path/to/repo``). + label: + Branch (or tag / SHA) to check out. Defaults to ``"main"``. + clone_dir: + Where to clone the repository. When *None* a temporary directory is + created automatically and cleaned up when the process exits. + """ + + def __init__( + self, + uri: str, + *, + label: str = "main", + clone_dir: str | None = None, + ) -> None: + self._uri = uri + self._label = label + self._clone_dir = clone_dir + self._repo: Any = None # git.Repo, set on first _ensure_repo() call + self._fs: FilesystemConfigBackend | None = None + + # ------------------------------------------------------------------ + # Internal helpers + # ------------------------------------------------------------------ + + def _ensure_repo(self) -> FilesystemConfigBackend: + """Lazily clone the repo and return (or return the cached) FS backend.""" + if self._fs is not None: + return self._fs + + git = _require_git() + + work_dir = self._clone_dir or tempfile.mkdtemp(prefix="pyfly-git-config-") + _logger.debug("GitConfigBackend: cloning %s → %s (label=%s)", self._uri, work_dir, self._label) + self._repo = git.Repo.clone_from(self._uri, work_dir) + # Checkout the requested label (branch / tag / sha). + try: + self._repo.git.checkout(self._label) + except git.GitCommandError as exc: + _logger.warning("GitConfigBackend: could not checkout %r: %s", self._label, exc) + + self._fs = FilesystemConfigBackend(work_dir) + return self._fs + + async def _run_sync(self, fn: Any, *args: Any) -> Any: + """Run a synchronous callable in the default executor.""" + loop = asyncio.get_event_loop() + return await loop.run_in_executor(None, fn, *args) + + async def _ensure_repo_async(self) -> FilesystemConfigBackend: + """Async wrapper that returns a typed FilesystemConfigBackend.""" + result: FilesystemConfigBackend = await self._run_sync(self._ensure_repo) + return result + + # ------------------------------------------------------------------ + # ConfigBackend protocol + # ------------------------------------------------------------------ + + async def fetch(self, application: str, profile: str, label: str = "main") -> ConfigSource | None: + fs = await self._ensure_repo_async() + return await fs.fetch(application, profile, label) + + async def save(self, source: ConfigSource) -> None: + """Write config into the working tree and create a local Git commit. + + .. note:: + Only a local commit is created. Pushing to the remote is **out of + scope** — call :meth:`refresh` to pull and :meth:`save` to write, + then push manually if needed. + """ + git = _require_git() + + fs = await self._ensure_repo_async() + await fs.save(source) + + repo: Any = self._repo + + def _commit() -> None: + # Stage all modified / new files under the work tree. + repo.git.add(A=True) + if not repo.index.diff("HEAD") and not repo.untracked_files: + _logger.debug("GitConfigBackend.save: nothing to commit") + return + commit_msg = f"pyfly: update {source.application}/{source.profile}@{source.label}" + try: + repo.index.commit(commit_msg) + except git.GitCommandError as exc: + _logger.warning("GitConfigBackend.save: commit failed: %s", exc) + + await self._run_sync(_commit) + + async def list(self) -> list[ConfigSource]: + fs = await self._ensure_repo_async() + return await fs.list() + + # ------------------------------------------------------------------ + # Git-specific extra + # ------------------------------------------------------------------ + + async def refresh(self) -> None: + """Pull the latest commits from ``origin`` (no-op when no remote exists).""" + if importlib.util.find_spec("git") is None: + msg = "GitConfigBackend requires GitPython — `pip install pyfly[config-server-git]`" + raise ImportError(msg) + + await self._ensure_repo_async() # ensures self._repo is set + repo: Any = self._repo + if not repo.remotes: + _logger.debug("GitConfigBackend.refresh: no remotes — skipping pull") + return + + def _pull() -> None: + repo.remotes.origin.pull() + + _logger.debug("GitConfigBackend.refresh: pulling from origin") + await self._run_sync(_pull) diff --git a/tests/config_server/test_git_backend.py b/tests/config_server/test_git_backend.py new file mode 100644 index 00000000..b4e08e0e --- /dev/null +++ b/tests/config_server/test_git_backend.py @@ -0,0 +1,169 @@ +# Copyright 2026 Firefly Software Foundation. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Tests for GitConfigBackend. + +All tests use a local (on-disk) bare/non-bare repository — no network access. +The suite is skipped automatically when GitPython is not installed. +""" + +from __future__ import annotations + +import pathlib + +import pytest + +git = pytest.importorskip("git") + +import yaml # noqa: E402 + +from pyfly.config_server.adapters.git import GitConfigBackend # noqa: E402 +from pyfly.config_server.backend import ConfigSource # noqa: E402 + + +def _make_repo(tmp_path: pathlib.Path, branch: str = "main") -> pathlib.Path: + """Create a local git repository with one committed config file.""" + repo_dir = tmp_path / "origin" + repo_dir.mkdir() + + repo = git.Repo.init(repo_dir) + # git requires at least name + email to commit. + repo.config_writer().set_value("user", "name", "Test").release() + repo.config_writer().set_value("user", "email", "test@example.com").release() + + # Write an orders-prod.yaml on the repo root (label "main" maps to root + # in _path_candidates_for, but FilesystemConfigBackend also checks the + # label sub-directory; place it at root for simplicity). + orders_yaml = repo_dir / "orders-prod.yaml" + orders_yaml.write_text(yaml.safe_dump({"db.url": "postgres://prod", "workers": 4})) + + repo.index.add(["orders-prod.yaml"]) + repo.index.commit("initial config") + + # Rename default branch to the requested name (git may default to + # "master" depending on the system git config). + if repo.active_branch.name != branch: + repo.head.reference = repo.create_head(branch) + repo.head.reset(index=True, working_tree=True) + + return repo_dir + + +@pytest.mark.asyncio +async def test_git_backend_fetch(tmp_path: pathlib.Path) -> None: + """GitConfigBackend.fetch reads properties committed in the repo.""" + origin = _make_repo(tmp_path) + clone_dir = str(tmp_path / "clone") + + backend = GitConfigBackend(str(origin), label="main", clone_dir=clone_dir) + source = await backend.fetch("orders", "prod") + + assert source is not None + assert source.application == "orders" + assert source.profile == "prod" + assert source.properties["db.url"] == "postgres://prod" + assert source.properties["workers"] == 4 + + +@pytest.mark.asyncio +async def test_git_backend_list(tmp_path: pathlib.Path) -> None: + """GitConfigBackend.list enumerates committed config files.""" + origin = _make_repo(tmp_path) + clone_dir = str(tmp_path / "clone") + + backend = GitConfigBackend(str(origin), label="main", clone_dir=clone_dir) + sources = await backend.list() + + assert any(s.application == "orders" and s.profile == "prod" for s in sources) + + +@pytest.mark.asyncio +async def test_git_backend_save_commits(tmp_path: pathlib.Path) -> None: + """GitConfigBackend.save writes the file and creates a local commit.""" + origin = _make_repo(tmp_path) + clone_dir = str(tmp_path / "clone") + + backend = GitConfigBackend(str(origin), label="main", clone_dir=clone_dir) + # First fetch to initialise the clone. + await backend.fetch("orders", "prod") + + new_source = ConfigSource( + application="payments", + profile="prod", + label="main", + properties={"gateway": "stripe", "retries": 3}, + ) + await backend.save(new_source) + + # The property is now readable through a fresh fetch. + fetched = await backend.fetch("payments", "prod") + assert fetched is not None + assert fetched.properties["gateway"] == "stripe" + + # A commit must have been created. + import git as gitlib + + repo = gitlib.Repo(clone_dir) + last_msg = repo.head.commit.message + assert "payments" in last_msg or "pyfly" in last_msg + + +@pytest.mark.asyncio +async def test_git_backend_save_updates_existing(tmp_path: pathlib.Path) -> None: + """GitConfigBackend.save updates an existing file (not creates a duplicate).""" + origin = _make_repo(tmp_path) + clone_dir = str(tmp_path / "clone") + + backend = GitConfigBackend(str(origin), label="main", clone_dir=clone_dir) + + updated = ConfigSource( + application="orders", + profile="prod", + label="main", + properties={"db.url": "postgres://prod-v2", "workers": 8}, + ) + await backend.save(updated) + + fetched = await backend.fetch("orders", "prod") + assert fetched is not None + assert fetched.properties["db.url"] == "postgres://prod-v2" + assert fetched.properties["workers"] == 8 + + +@pytest.mark.asyncio +async def test_git_backend_refresh_no_remote(tmp_path: pathlib.Path) -> None: + """refresh() is a no-op (not an error) when the clone has no remote.""" + origin = _make_repo(tmp_path) + clone_dir = str(tmp_path / "clone") + + backend = GitConfigBackend(str(origin), label="main", clone_dir=clone_dir) + await backend.fetch("orders", "prod") # init clone + + # Remove the remote so refresh() must skip gracefully. + import git as gitlib + + repo = gitlib.Repo(clone_dir) + repo.delete_remote(repo.remote("origin")) + + await backend.refresh() # must not raise + + +@pytest.mark.asyncio +async def test_git_backend_missing_returns_none(tmp_path: pathlib.Path) -> None: + """fetch() returns None for a file that doesn't exist in the repo.""" + origin = _make_repo(tmp_path) + clone_dir = str(tmp_path / "clone") + + backend = GitConfigBackend(str(origin), label="main", clone_dir=clone_dir) + result = await backend.fetch("nonexistent", "dev") + assert result is None diff --git a/uv.lock b/uv.lock index e22ada86..1e7296b8 100644 --- a/uv.lock +++ b/uv.lock @@ -846,6 +846,30 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/4c/a0/614c5fe402fd88951df45f4dda2fa3b4e17a99ecd92340771929169b3b95/filelock-3.29.1-py3-none-any.whl", hash = "sha256:85199dfd706869641b72b2e8955d5416a4b2b7dc4b0e8e6d97b4cc1299a6983b", size = 40750, upload-time = "2026-06-03T15:19:02.959Z" }, ] +[[package]] +name = "gitdb" +version = "4.0.12" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "smmap" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/72/94/63b0fc47eb32792c7ba1fe1b694daec9a63620db1e313033d18140c2320a/gitdb-4.0.12.tar.gz", hash = "sha256:5ef71f855d191a3326fcfbc0d5da835f26b13fbcba60c32c21091c349ffdb571", size = 394684, upload-time = "2025-01-02T07:20:46.413Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a0/61/5c78b91c3143ed5c14207f463aecfc8f9dbb5092fb2869baf37c273b2705/gitdb-4.0.12-py3-none-any.whl", hash = "sha256:67073e15955400952c6565cc3e707c554a4eea2e428946f7a4c162fab9bd9bcf", size = 62794, upload-time = "2025-01-02T07:20:43.624Z" }, +] + +[[package]] +name = "gitpython" +version = "3.1.50" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "gitdb" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/33/f6/354ae6491228b5eb40e10d89c4d13c651fe1cf7556e35ebdded50cff57ce/gitpython-3.1.50.tar.gz", hash = "sha256:80da2d12504d52e1f998772dc5baf6e553f8d2fcfe1fcc226c9d9a2ee3372dcc", size = 219798, upload-time = "2026-05-06T04:01:26.571Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/20/7a/1c6e3562dfd8950adbb11ffbc65d21e7c89d01a6e4f137fa981056de25c5/gitpython-3.1.50-py3-none-any.whl", hash = "sha256:d352abe2908d07355014abdd21ddf798c2a961469239afec4962e9da884858f9", size = 212507, upload-time = "2026-05-06T04:01:23.799Z" }, +] + [[package]] name = "granian" version = "2.7.4" @@ -2156,6 +2180,9 @@ cli = [ client = [ { name = "httpx" }, ] +config-server-git = [ + { name = "gitpython" }, +] data-document = [ { name = "beanie" }, ] @@ -2191,6 +2218,7 @@ full = [ { name = "croniter" }, { name = "cryptography" }, { name = "fastapi" }, + { name = "gitpython" }, { name = "granian" }, { name = "grpcio" }, { name = "httpx" }, @@ -2329,6 +2357,7 @@ requires-dist = [ { name = "croniter", marker = "extra == 'scheduling'", specifier = ">=6.2.2" }, { name = "cryptography", marker = "extra == 'security'", specifier = ">=48.0.0" }, { name = "fastapi", marker = "extra == 'fastapi'", specifier = ">=0.136.1" }, + { name = "gitpython", marker = "extra == 'config-server-git'", specifier = ">=3.1" }, { name = "granian", marker = "extra == 'granian'", specifier = ">=2.7.4" }, { name = "grpcio", marker = "extra == 'grpc'", specifier = ">=1.60.0" }, { name = "httpx", marker = "extra == 'client'", specifier = ">=0.28.1" }, @@ -2347,7 +2376,7 @@ requires-dist = [ { name = "prometheus-client", marker = "extra == 'observability'", specifier = ">=0.25.0" }, { name = "pydantic", specifier = ">=2.13.3" }, { name = "pyfly", extras = ["fastapi", "granian"], marker = "extra == 'web-fastapi'" }, - { name = "pyfly", extras = ["web", "data-relational", "data-document", "postgresql", "eda", "cache", "client", "grpc", "websocket", "ecm-aws", "ecm-azure", "observability", "security", "scheduling", "cli", "shell", "kafka", "rabbitmq", "redis", "granian", "fastapi", "hypercorn", "idp-azure", "idp-keycloak", "idp-cognito", "notifications"], marker = "extra == 'full'" }, + { name = "pyfly", extras = ["web", "data-relational", "data-document", "postgresql", "eda", "cache", "client", "grpc", "websocket", "ecm-aws", "ecm-azure", "observability", "security", "scheduling", "cli", "shell", "kafka", "rabbitmq", "redis", "granian", "fastapi", "hypercorn", "idp-azure", "idp-keycloak", "idp-cognito", "notifications", "config-server-git"], marker = "extra == 'full'" }, { name = "pyfly", extras = ["web", "granian"], marker = "extra == 'web-fast'" }, { name = "pyjwt", extras = ["crypto"], marker = "extra == 'security'", specifier = ">=2.12.1" }, { name = "pyotp", marker = "extra == 'security'", specifier = ">=2.9.0" }, @@ -2366,7 +2395,7 @@ requires-dist = [ { name = "uvloop", marker = "sys_platform != 'win32' and extra == 'web-fastapi'", specifier = ">=0.22.1" }, { name = "websockets", marker = "extra == 'websocket'", specifier = ">=12.0" }, ] -provides-extras = ["web", "data-relational", "testing", "testcontainers", "data-document", "postgresql", "eda", "fastapi", "granian", "hypercorn", "kafka", "rabbitmq", "redis", "cache", "client", "grpc", "websocket", "idp-azure", "idp-keycloak", "idp-cognito", "ecm-aws", "ecm-azure", "observability", "scheduling", "pii", "security", "notifications", "cli", "shell", "web-fast", "web-fastapi", "full"] +provides-extras = ["web", "data-relational", "testing", "testcontainers", "data-document", "postgresql", "eda", "fastapi", "granian", "hypercorn", "kafka", "rabbitmq", "redis", "cache", "client", "config-server-git", "grpc", "websocket", "idp-azure", "idp-keycloak", "idp-cognito", "ecm-aws", "ecm-azure", "observability", "scheduling", "pii", "security", "notifications", "cli", "shell", "web-fast", "web-fastapi", "full"] [package.metadata.requires-dev] dev = [ @@ -2863,6 +2892,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/14/78/0f68b93564b8c6b6987a0696c582ba2591a381ab2f733a501909e949f241/smart_open-7.6.1-py3-none-any.whl", hash = "sha256:b4de6aebef023aca91cc9fb372052e1343ba3f152de215bd22391a663e3ddd21", size = 64845, upload-time = "2026-05-09T06:23:35.386Z" }, ] +[[package]] +name = "smmap" +version = "5.0.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/1f/ea/49c993d6dfdd7338c9b1000a0f36817ed7ec84577ae2e52f890d1a4ff909/smmap-5.0.3.tar.gz", hash = "sha256:4d9debb8b99007ae47165abc08670bd74cb74b5227dda7f643eccc4e9eb5642c", size = 22506, upload-time = "2026-03-09T03:43:26.1Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c1/d4/59e74daffcb57a07668852eeeb6035af9f32cbfd7a1d2511f17d2fe6a738/smmap-5.0.3-py3-none-any.whl", hash = "sha256:c106e05d5a61449cf6ba9a1e650227ecfb141590d2a98412103ff35d89fc7b2f", size = 24390, upload-time = "2026-03-09T03:43:24.361Z" }, +] + [[package]] name = "spacy" version = "3.8.14" From a5e541c56771d671b36e92c39f7d92a2046a05f0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9s=20Contreras=20Guill=C3=A9n?= Date: Wed, 10 Jun 2026 02:06:51 +0200 Subject: [PATCH 2/6] feat(config-server): make ConfigClient transport-injectable + ASGITransport e2e test MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `client.py`: adds optional `http_client: Any | None = None` ctor param. When injected, `fetch()` uses it without closing (caller owns lifecycle); when omitted, builds+closes its own httpx.AsyncClient as before. `tests/config_server/test_config_client_e2e.py`: three async tests that drive `ConfigClient` through `httpx.ASGITransport(app=starlette_app)` with no mocking — a real round-trip through routing, JSON serialisation, and reverse-merge precedence (app+profile beats application+default; common-only key survives). --- src/pyfly/config_server/client.py | 37 +++-- tests/config_server/test_config_client_e2e.py | 144 ++++++++++++++++++ 2 files changed, 171 insertions(+), 10 deletions(-) create mode 100644 tests/config_server/test_config_client_e2e.py diff --git a/src/pyfly/config_server/client.py b/src/pyfly/config_server/client.py index 5df03f52..a8d8f378 100644 --- a/src/pyfly/config_server/client.py +++ b/src/pyfly/config_server/client.py @@ -11,7 +11,14 @@ class ConfigClient: - """Minimal HTTP client for a Spring-Cloud-Config-style server.""" + """Minimal HTTP client for a Spring-Cloud-Config-style server. + + An optional *http_client* (an ``httpx.AsyncClient`` instance) may be + injected for testing or connection-pool sharing. When provided the caller + owns the client's lifecycle — it is **not** closed by :meth:`fetch`. When + omitted, a fresh ``httpx.AsyncClient`` is created and closed per call as + before (unchanged public behaviour). + """ def __init__( self, @@ -22,6 +29,7 @@ def __init__( label: str = "main", username: str | None = None, password: str | None = None, + http_client: Any | None = None, ) -> None: self._url = url.rstrip("/") self._application = application @@ -29,6 +37,7 @@ def __init__( self._label = label self._username = username self._password = password + self._http_client = http_client async def fetch(self) -> dict[str, Any]: try: @@ -41,7 +50,8 @@ async def fetch(self) -> dict[str, Any]: auth: tuple[str, str] | None = ( (self._username, self._password) if self._username is not None and self._password is not None else None ) - async with httpx.AsyncClient(timeout=15.0) as client: + + async def _do_get(client: Any) -> dict[str, Any]: resp = await client.get(path, auth=auth) if resp.status_code != 200: _logger.warning( @@ -52,11 +62,18 @@ async def fetch(self) -> dict[str, Any]: self._label, ) return {} - data = resp.json() - # Spring orders propertySources HIGHEST priority first, so apply them - # in reverse (lowest first) and let higher-priority sources overwrite — - # the forward order let the lowest-priority source win (audit #86). - merged: dict[str, Any] = {} - for source in reversed(data.get("propertySources", [])): - merged.update(source.get("source") or {}) - return merged + data: dict[str, Any] = resp.json() + # Spring orders propertySources HIGHEST priority first, so apply them + # in reverse (lowest first) and let higher-priority sources overwrite — + # the forward order let the lowest-priority source win (audit #86). + merged: dict[str, Any] = {} + for source in reversed(data.get("propertySources", [])): + merged.update(source.get("source") or {}) + return merged + + if self._http_client is not None: + # Injected client — caller owns lifecycle; do NOT close it. + return await _do_get(self._http_client) + + async with httpx.AsyncClient(timeout=15.0) as client: + return await _do_get(client) diff --git a/tests/config_server/test_config_client_e2e.py b/tests/config_server/test_config_client_e2e.py new file mode 100644 index 00000000..a30e377d --- /dev/null +++ b/tests/config_server/test_config_client_e2e.py @@ -0,0 +1,144 @@ +# Copyright 2026 Firefly Software Foundation. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""End-to-end test: ConfigClient → Starlette (ASGITransport) → ConfigServer. + +The test uses ``httpx.ASGITransport`` so there is no network socket involved, +but the full request/response path — HTTP routing, JSON serialisation, and +``ConfigClient.fetch()``'s reverse-merge of ``propertySources`` — is exercised +for real. Nothing is mocked. +""" + +from __future__ import annotations + +import pytest + +httpx = pytest.importorskip("httpx") +starlette = pytest.importorskip("starlette") + +from starlette.applications import Starlette # noqa: E402 + +from pyfly.config_server.adapters.starlette import make_starlette_config_server_routes # noqa: E402 +from pyfly.config_server.backend import ConfigSource, InMemoryConfigBackend # noqa: E402 +from pyfly.config_server.client import ConfigClient # noqa: E402 +from pyfly.config_server.server import ConfigServer # noqa: E402 + + +@pytest.mark.asyncio +async def test_e2e_reverse_merge_precedence() -> None: + """ConfigClient merges propertySources in reverse so app+profile wins. + + Seed layout + ----------- + ``orders / prod`` → ``{"host": "prod.db", "port": 5432, "timeout": 30}`` + ``application / default`` → ``{"host": "default.db", "port": 5432, "shared_key": "common"}`` + + Expected merged result (app+profile overrides application+default): + - ``host`` == ``"prod.db"`` (app+profile wins over application+default) + - ``port`` == 5432 (same in both — no conflict) + - ``timeout`` == 30 (only in app+profile) + - ``shared_key`` == ``"common"`` (only in application+default — survives) + """ + # 1. Build the in-process config server. + backend = InMemoryConfigBackend() + await backend.save( + ConfigSource( + application="orders", + profile="prod", + label="main", + properties={"host": "prod.db", "port": 5432, "timeout": 30}, + ) + ) + await backend.save( + ConfigSource( + application="application", + profile="default", + label="main", + properties={"host": "default.db", "port": 5432, "shared_key": "common"}, + ) + ) + server = ConfigServer(backend=backend) + app = Starlette(routes=make_starlette_config_server_routes(server)) + + # 2. Create an httpx client wired directly to the ASGI app — no socket. + transport = httpx.ASGITransport(app=app) # type: ignore[arg-type] + async with httpx.AsyncClient(transport=transport, base_url="http://config") as http_client: + client = ConfigClient( + url="http://config", + application="orders", + profile="prod", + label="main", + http_client=http_client, + ) + result = await client.fetch() + + # 3. Assert precedence is respected. + assert result["host"] == "prod.db", "app+profile must override application+default" + assert result["port"] == 5432 + assert result["timeout"] == 30, "key only in app+profile must survive" + assert result["shared_key"] == "common", "key only in application+default must be inherited" + + +@pytest.mark.asyncio +async def test_e2e_missing_config_returns_empty() -> None: + """A 404 from the server should yield an empty dict, not raise.""" + backend = InMemoryConfigBackend() + server = ConfigServer(backend=backend) + app = Starlette(routes=make_starlette_config_server_routes(server)) + + transport = httpx.ASGITransport(app=app) # type: ignore[arg-type] + async with httpx.AsyncClient(transport=transport, base_url="http://config") as http_client: + client = ConfigClient( + url="http://config", + application="missing", + profile="ghost", + label="main", + http_client=http_client, + ) + result = await client.fetch() + + assert result == {} + + +@pytest.mark.asyncio +async def test_e2e_injected_client_not_closed() -> None: + """The injected http_client must not be closed after fetch().""" + backend = InMemoryConfigBackend() + await backend.save( + ConfigSource( + application="svc", + profile="default", + label="main", + properties={"k": "v"}, + ) + ) + server = ConfigServer(backend=backend) + app = Starlette(routes=make_starlette_config_server_routes(server)) + + transport = httpx.ASGITransport(app=app) # type: ignore[arg-type] + http_client = httpx.AsyncClient(transport=transport, base_url="http://config") + try: + client = ConfigClient( + url="http://config", + application="svc", + profile="default", + label="main", + http_client=http_client, + ) + result = await client.fetch() + assert result == {"k": "v"} + # Client still open — a second fetch must succeed. + result2 = await client.fetch() + assert result2 == {"k": "v"} + finally: + await http_client.aclose() From 417114b249cb02f14c304bf399a8bfe95cfc13c2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9s=20Contreras=20Guill=C3=A9n?= Date: Wed, 10 Jun 2026 02:06:58 +0200 Subject: [PATCH 3/6] feat(config-server): add tiered search_locations to FilesystemConfigBackend `backend.py`: `FilesystemConfigBackend` accepts optional `search_locations: list[str | Path] | None = None` (highest-precedence first, e.g. [domain, core, common]). When set, `fetch()` iterates locations from lowest to highest and merges properties so higher-precedence keys win and lower-precedence fills gaps. `save()` and `list()` always target `_root` (the primary location). Single-root behaviour unchanged when None. `tests/config_server/test_tiered_overlay.py`: seven tests covering domain-overrides-common, three-tier chain, missing-in-all, partial presence, save-to-primary, list-uses-primary, and single-root-unchanged. --- src/pyfly/config_server/backend.py | 78 ++++++++++- tests/config_server/test_tiered_overlay.py | 154 +++++++++++++++++++++ 2 files changed, 231 insertions(+), 1 deletion(-) create mode 100644 tests/config_server/test_tiered_overlay.py diff --git a/src/pyfly/config_server/backend.py b/src/pyfly/config_server/backend.py index d585c439..e0828ef2 100644 --- a/src/pyfly/config_server/backend.py +++ b/src/pyfly/config_server/backend.py @@ -53,11 +53,37 @@ class FilesystemConfigBackend: """Loads config from ``/-.{yaml,yml,json}``. The label maps to a subdirectory: ``/