From 377f665cf1de8497bf6bc6e81618b8883c13d4e7 Mon Sep 17 00:00:00 2001 From: Jimisola Laursen Date: Mon, 1 Jun 2026 17:06:22 +0200 Subject: [PATCH 1/5] refactor(locations): replace env_token with direct-value token field MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Rename env_token → token across all location types (git, maven, pypi, npm) and change semantics: the field now holds the token value directly rather than an environment variable name. Users supply tokens via \${VAR} interpolation introduced in #397 (e.g. token: \${GITLAB_TOKEN}). Removes os.getenv() indirection from all location classes. Updates JSON schema, generated Pydantic models, CLI flag (--env_token → --token), model generator, all test fixtures, and unit / integration tests accordingly. Signed-off-by: Jimisola Laursen --- .../requirements/requirements_microservice.yml | 6 +++--- .../requirements/requirements_system.yml | 4 ++-- src/reqstool/command.py | 16 ++++++++-------- src/reqstool/locations/git_location.py | 5 ++--- src/reqstool/locations/maven_location.py | 5 ++--- src/reqstool/locations/npm_location.py | 7 +++---- src/reqstool/locations/pypi_location.py | 11 +++-------- .../requirements_model_generator.py | 8 ++++---- .../models/generated/requirements_schema.py | 16 ++++++++-------- .../schemas/v1/requirements.schema.json | 16 ++++++++-------- .../test_included_models_generator.py | 11 ++++------- .../test_regression_monorepo.py | 4 +++- .../test_regression_structure.py | 4 +++- .../sys-001/requirements.yml | 2 +- .../requirements.yml | 4 ++-- .../requirements.yml | 2 +- .../requirements.yml | 12 ++++++------ tests/unit/reqstool/common/test_locations.py | 8 ++++---- .../reqstool/locations/test_git_location.py | 12 ++++++------ .../reqstool/locations/test_maven_location.py | 6 +++--- .../reqstool/locations/test_npm_location.py | 12 +++++------- .../reqstool/locations/test_pypi_location.py | 6 +++--- .../test_requirements_model_generator.py | 18 +++++++++--------- .../reqstool/models/test_implementations.py | 8 ++++---- tests/unit/reqstool/models/test_imports.py | 8 ++++---- tests/unit/reqstool/test_command.py | 10 +++++----- 26 files changed, 106 insertions(+), 115 deletions(-) diff --git a/docs/modules/examples/partials/requirements/requirements_microservice.yml b/docs/modules/examples/partials/requirements/requirements_microservice.yml index 956277bc..d8b47503 100644 --- a/docs/modules/examples/partials/requirements/requirements_microservice.yml +++ b/docs/modules/examples/partials/requirements/requirements_microservice.yml @@ -8,19 +8,19 @@ metadata: imports: #Optional git: - - env_token: e.g. GITLAB_TOKEN or empty + - token: ${GITLAB_TOKEN} # use ${VAR} interpolation to supply from environment ref: url: url path: path maven: - - env_token: e.g. MAVEN_TOKEN or empty + - token: ${MAVEN_TOKEN} # use ${VAR} interpolation to supply from environment url: x.se group_id: x artifact_id: y classifier: s version: 1.0.0 pypi: - - env_token: e.g. PYPI_TOKEN or empty + - token: ${PYPI_TOKEN} # use ${VAR} interpolation to supply from environment url: url package: package-name version: 1.0.0 diff --git a/docs/modules/examples/partials/requirements/requirements_system.yml b/docs/modules/examples/partials/requirements/requirements_system.yml index 2e64a3d6..9cdf5cb7 100644 --- a/docs/modules/examples/partials/requirements/requirements_system.yml +++ b/docs/modules/examples/partials/requirements/requirements_system.yml @@ -8,7 +8,7 @@ metadata: imports: git: - - env_token: e.g. GITLAB_TOKEN or empty + - token: ${GITLAB_TOKEN} # use ${VAR} interpolation to supply from environment ref: url: path: @@ -26,7 +26,7 @@ filters: implementations: # only used with type = system git: - - env_token: e.g. GITLAB_TOKEN or empty + - token: ${GITLAB_TOKEN} # use ${VAR} interpolation to supply from environment ref: url: path: diff --git a/src/reqstool/command.py b/src/reqstool/command.py index b9e623be..f5953b01 100755 --- a/src/reqstool/command.py +++ b/src/reqstool/command.py @@ -74,7 +74,7 @@ {"flags": ["-u", "--url"], "kwargs": {"help": "git repository URL", "required": True}}, {"flags": ["-p", "--path"], "kwargs": {"help": "path within the repository", "required": True}}, {"flags": ["-r", "--ref"], "kwargs": {"help": "git branch, tag, or commit SHA", "required": True}}, - {"flags": ["-t", "--env_token"], "kwargs": {"help": "env var name holding the access token"}}, + {"flags": ["-t", "--token"], "kwargs": {"help": "authentication token value (use $VAR to read from environment)"}}, ], }, { @@ -82,7 +82,7 @@ "help": "maven source", "args": [ {"flags": ["-u", "--url"], "kwargs": {"help": "Maven repository URL", "required": False}}, - {"flags": ["-t", "--env_token"], "kwargs": {"help": "env var name holding the access token"}}, + {"flags": ["-t", "--token"], "kwargs": {"help": "authentication token value (use $VAR to read from environment)"}}, {"flags": ["--group_id"], "kwargs": {"help": "Maven group ID", "required": True}}, {"flags": ["--artifact_id"], "kwargs": {"help": "Maven artifact ID", "required": True}}, {"flags": ["--version"], "kwargs": {"help": "artifact version (e.g. 1.2.3)", "required": True}}, @@ -100,7 +100,7 @@ "required": False, }, }, - {"flags": ["-t", "--env_token"], "kwargs": {"help": "env var name holding the Bearer token"}}, + {"flags": ["-t", "--token"], "kwargs": {"help": "authentication token value (use $VAR to read from environment)"}}, {"flags": ["--package"], "kwargs": {"help": "npm package name (e.g. @scope/package)", "required": True}}, {"flags": ["--version"], "kwargs": {"help": "package version (e.g. 1.2.3)", "required": True}}, ], @@ -110,7 +110,7 @@ "help": "pypi source", "args": [ {"flags": ["-u", "--url"], "kwargs": {"help": "PyPI index URL", "required": False}}, - {"flags": ["-t", "--env_token"], "kwargs": {"help": "env var name holding the access token"}}, + {"flags": ["-t", "--token"], "kwargs": {"help": "authentication token value (use $VAR to read from environment)"}}, {"flags": ["--package"], "kwargs": {"help": "PyPI package name", "required": True}}, {"flags": ["--version"], "kwargs": {"help": "package version (e.g. 1.2.3)", "required": True}}, ], @@ -437,28 +437,28 @@ def _get_initial_source(self, args_source: argparse.Namespace) -> LocationInterf artifact_id=args_source.artifact_id, version=args_source.version, classifier=args_source.classifier if args_source.classifier else None, - env_token=args_source.env_token if args_source.env_token else None, + token=args_source.token if args_source.token else None, ) elif args_source.source == "npm": location = NpmLocation( url=args_source.url if args_source.url else "https://registry.npmjs.org", package=args_source.package, version=args_source.version, - env_token=args_source.env_token if args_source.env_token else None, + token=args_source.token if args_source.token else None, ) elif args_source.source == "pypi": location = PypiLocation( url=args_source.url if args_source.url else None, package=args_source.package, version=args_source.version, - env_token=args_source.env_token if args_source.env_token else None, + token=args_source.token if args_source.token else None, ) elif args_source.source == "git": location = GitLocation( url=args_source.url, path=args_source.path, ref=args_source.ref, - env_token=args_source.env_token if args_source.env_token else None, + token=args_source.token if args_source.token else None, ) elif args_source.source == "local": if args_source.maven: diff --git a/src/reqstool/locations/git_location.py b/src/reqstool/locations/git_location.py index 22fe5719..bd067dcf 100644 --- a/src/reqstool/locations/git_location.py +++ b/src/reqstool/locations/git_location.py @@ -1,7 +1,6 @@ # Copyright © LFV import logging -import os import re from typing import Optional @@ -21,7 +20,7 @@ class GitLocation(LocationInterface): url: str ref: str - env_token: Optional[str] = None + token: Optional[str] = None path: str = "" @field_validator("ref") @@ -44,7 +43,7 @@ def tmpdir_key(self) -> str: return make_safe_tmpdir_suffix("git", f"{urlunparse(parsed)}@{self.ref}") def _make_available_on_localdisk(self, dst_path: str) -> str: - api_token = os.getenv(self.env_token) if self.env_token else None + api_token = self.token repo = clone_repository(url=self.url, path=dst_path, callbacks=self.MyRemoteCallbacks(api_token)) diff --git a/src/reqstool/locations/maven_location.py b/src/reqstool/locations/maven_location.py index 51d4614b..30beaea3 100644 --- a/src/reqstool/locations/maven_location.py +++ b/src/reqstool/locations/maven_location.py @@ -1,7 +1,6 @@ # Copyright © LFV import logging -import os from typing import Optional from maven_artifact import Artifact, Downloader, RequestException @@ -19,13 +18,13 @@ class MavenLocation(LocationInterface): artifact_id: str version: str classifier: str = "reqstool" - env_token: Optional[str] = None + token: Optional[str] = None def tmpdir_key(self) -> str: return make_safe_tmpdir_suffix("maven", f"{self.group_id}:{self.artifact_id}:{self.version}") def _make_available_on_localdisk(self, dst_path: str): - token = os.getenv(self.env_token) if self.env_token else None + token = self.token # assume OAuth Bearer, see: https://georgearisty.dev/posts/oauth2-token-bearer-usage/ downloader = Downloader(base=self.url, token=token) diff --git a/src/reqstool/locations/npm_location.py b/src/reqstool/locations/npm_location.py index 2db09541..d2c343ea 100644 --- a/src/reqstool/locations/npm_location.py +++ b/src/reqstool/locations/npm_location.py @@ -1,7 +1,6 @@ # Copyright © LFV import logging -import os import tarfile from typing import Optional from urllib.parse import quote, urlparse @@ -21,7 +20,7 @@ class NpmLocation(LocationInterface): url: str = "https://registry.npmjs.org" package: str version: str - env_token: Optional[str] = None + token: Optional[str] = None @field_validator("url") @classmethod @@ -34,8 +33,8 @@ def tmpdir_key(self) -> str: return make_safe_tmpdir_suffix("npm", f"{self.package}@{self.version}") def _make_available_on_localdisk(self, dst_path: str): - """Resolve token → fetch tarball URL → SSRF check → download → extract.""" - token = os.getenv(self.env_token) if self.env_token else None + """Fetch tarball URL → SSRF check → download → extract.""" + token = self.token try: tarball_url = self._get_tarball_url(token) diff --git a/src/reqstool/locations/pypi_location.py b/src/reqstool/locations/pypi_location.py index dec3b398..4f51e146 100644 --- a/src/reqstool/locations/pypi_location.py +++ b/src/reqstool/locations/pypi_location.py @@ -1,5 +1,4 @@ import logging -import os import re import tarfile from typing import Optional @@ -16,7 +15,7 @@ class PypiLocation(LocationInterface): url: str = "https://pypi.org/simple" package: str version: str - env_token: Optional[str] = None + token: Optional[str] = None @staticmethod def normalize_pypi_package_name(package_name): @@ -26,11 +25,7 @@ def tmpdir_key(self) -> str: return make_safe_tmpdir_suffix("pypi", f"{self.package}=={self.version}") def _make_available_on_localdisk(self, dst_path: str): - """ - Download the PyPI package and extract it to the local disk. - """ - # Retrieve token from environment variable - token = os.getenv(self.env_token) if self.env_token else None + token = self.token if token: logging.debug("Using OAuth Bearer token for authentication") @@ -38,7 +33,7 @@ def _make_available_on_localdisk(self, dst_path: str): package_url = self.get_package_url(self.package, self.version, self.url, token) if not package_url: - token_info = f"(with token in environment variable '{self.env_token}')" if self.env_token else "" + token_info = "(with authentication token)" if self.token else "" raise RuntimeError( f"Unable to find a sdist pypi package for {self.package} == {self.version} in repo {self.url}{token_info}" ) diff --git a/src/reqstool/model_generators/requirements_model_generator.py b/src/reqstool/model_generators/requirements_model_generator.py index fb485989..71af6850 100644 --- a/src/reqstool/model_generators/requirements_model_generator.py +++ b/src/reqstool/model_generators/requirements_model_generator.py @@ -208,7 +208,7 @@ def __parse_location_maven(self, locations_obj, instance_type, locations): maven_location = instance_type( parent=self.parent, current_unresolved=MavenLocation( - env_token=maven.env_token, + token=maven.token, url=maven.url.root if maven.url else MAVEN_CENTRAL_REPO_URL, group_id=maven.group_id, artifact_id=maven.artifact_id, @@ -227,7 +227,7 @@ def __parse_location_pypi(self, locations_obj, instance_type, locations): pypi_location = instance_type( parent=self.parent, current_unresolved=PypiLocation( - env_token=pypi.env_token, + token=pypi.token, url=pypi.url.root if pypi.url else PYPI_ORG_SIMPLE_API_URL, package=pypi.package, version=pypi.version, @@ -239,7 +239,7 @@ def __parse_location_pypi(self, locations_obj, instance_type, locations): def __parse_location_npm(self, locations_obj, instance_type, locations): if locations_obj.npm is not None: for npm in locations_obj.npm: - npm_kwargs = {"env_token": npm.env_token, "package": npm.package, "version": npm.version} + npm_kwargs = {"token": npm.token, "package": npm.package, "version": npm.version} if npm.url: npm_kwargs["url"] = npm.url.root npm_location = instance_type( @@ -262,7 +262,7 @@ def __parse_location_git(self, locations_obj, instance_type, locations): git_location = instance_type( parent=self.parent, current_unresolved=GitLocation( - env_token=git.env_token, + token=git.token, url=git.url, ref=git.ref, path=git.path or "", diff --git a/src/reqstool/models/generated/requirements_schema.py b/src/reqstool/models/generated/requirements_schema.py index 4bcc83f7..fca7f496 100644 --- a/src/reqstool/models/generated/requirements_schema.py +++ b/src/reqstool/models/generated/requirements_schema.py @@ -51,9 +51,9 @@ class Git(BaseModel): model_config = ConfigDict( extra='forbid', ) - env_token: str | None = None + token: str | None = None """ - Token to authenticate. E.g. GITLAB_TOKEN or empty. + Authentication token value. Use ${VAR} interpolation to supply from environment, e.g. ${GITLAB_TOKEN}. """ ref: str """ @@ -211,9 +211,9 @@ class Maven(BaseModel): model_config = ConfigDict( extra='forbid', ) - env_token: str | None = None + token: str | None = None """ - Token to authenticate. E.g. MAVEN_TOKEN or empty. + Authentication token value. Use ${VAR} interpolation to supply from environment, e.g. ${MAVEN_TOKEN}. """ url: Url | None = None """ @@ -241,9 +241,9 @@ class Npm(BaseModel): model_config = ConfigDict( extra='forbid', ) - env_token: str | None = None + token: str | None = None """ - Token to authenticate. E.g. NPM_TOKEN or empty. + Authentication token value. Use ${VAR} interpolation to supply from environment, e.g. ${NPM_TOKEN}. """ url: Url | None = None """ @@ -263,9 +263,9 @@ class Pypi(BaseModel): model_config = ConfigDict( extra='forbid', ) - env_token: str | None = None + token: str | None = None """ - Token to authenticate. E.g. PYPI_TOKEN or empty. + Authentication token value. Use ${VAR} interpolation to supply from environment, e.g. ${PYPI_TOKEN}. """ url: Url | None = None """ diff --git a/src/reqstool/resources/schemas/v1/requirements.schema.json b/src/reqstool/resources/schemas/v1/requirements.schema.json index 7bc1b5e8..ded707fd 100644 --- a/src/reqstool/resources/schemas/v1/requirements.schema.json +++ b/src/reqstool/resources/schemas/v1/requirements.schema.json @@ -109,9 +109,9 @@ "type": "object", "additionalProperties": false, "properties": { - "env_token": { + "token": { "type": "string", - "description": "Token to authenticate. E.g. GITLAB_TOKEN or empty." + "description": "Authentication token value. Use ${VAR} interpolation to supply from environment, e.g. ${GITLAB_TOKEN}." }, "ref": { "type": "string", @@ -135,9 +135,9 @@ "type": "object", "additionalProperties": false, "properties": { - "env_token": { + "token": { "type": "string", - "description": "Token to authenticate. E.g. MAVEN_TOKEN or empty." + "description": "Authentication token value. Use ${VAR} interpolation to supply from environment, e.g. ${MAVEN_TOKEN}." }, "url": { "$ref": "#/$defs/url", @@ -171,9 +171,9 @@ "type": "object", "additionalProperties": false, "properties": { - "env_token": { + "token": { "type": "string", - "description": "Token to authenticate. E.g. NPM_TOKEN or empty." + "description": "Authentication token value. Use ${VAR} interpolation to supply from environment, e.g. ${NPM_TOKEN}." }, "url": { "$ref": "#/$defs/url", @@ -198,9 +198,9 @@ "type": "object", "additionalProperties": false, "properties": { - "env_token": { + "token": { "type": "string", - "description": "Token to authenticate. E.g. PYPI_TOKEN or empty." + "description": "Authentication token value. Use ${VAR} interpolation to supply from environment, e.g. ${PYPI_TOKEN}." }, "url": { "$ref": "#/$defs/url", diff --git a/tests/integration/reqstool/model_generators/test_included_models_generator.py b/tests/integration/reqstool/model_generators/test_included_models_generator.py index 9b0e5193..f375b852 100644 --- a/tests/integration/reqstool/model_generators/test_included_models_generator.py +++ b/tests/integration/reqstool/model_generators/test_included_models_generator.py @@ -12,11 +12,8 @@ from reqstool.model_generators import combined_raw_datasets_generator -def choose_token() -> str: - if os.getenv("GITHUB_TOKEN"): - return "GITHUB_TOKEN" - else: - return "GITLAB_TOKEN" +def choose_token(): + return os.getenv("GITHUB_TOKEN") or os.getenv("GITLAB_TOKEN") @SVCs("SVC_002") @@ -30,7 +27,7 @@ def test_basic_git(): combined_raw_datasets_generator.CombinedRawDatasetsGenerator( initial_location=GitLocation( - env_token=choose_token(), + token=choose_token(), url="https://github.com/reqstool/reqstool-client.git", path="tests/resources/test_data/data/remote/test_standard/test_standard_maven_git/ms-001", ref="main", @@ -51,7 +48,7 @@ def test_basic_maven(): combined_raw_datasets_generator.CombinedRawDatasetsGenerator( # Setup initial_location=MavenLocation( - env_token=choose_token(), + token=choose_token(), url="https://maven.pkg.github.com/reqstool/reqstool-demo", group_id="se.lfv.reqstool", artifact_id="reqstool-demo", diff --git a/tests/integration/reqstool/model_generators/test_regression_monorepo.py b/tests/integration/reqstool/model_generators/test_regression_monorepo.py index 42eac87f..871c87f6 100644 --- a/tests/integration/reqstool/model_generators/test_regression_monorepo.py +++ b/tests/integration/reqstool/model_generators/test_regression_monorepo.py @@ -1,5 +1,7 @@ # Copyright © reqstool +import os + import pytest from integration.reqstool.model_generators._regression_shared import ( @@ -28,7 +30,7 @@ def _make_generator(path: str, tmpdir_manager=None): holder = ValidationErrorHolder() gen = combined_raw_datasets_generator.CombinedRawDatasetsGenerator( initial_location=GitLocation( - env_token=_GITHUB_TOKEN_ENV, + token=os.getenv(_GITHUB_TOKEN_ENV), url=_REGRESSION_REPO_URL, ref=_REGRESSION_REPO_REF, path=path, diff --git a/tests/integration/reqstool/model_generators/test_regression_structure.py b/tests/integration/reqstool/model_generators/test_regression_structure.py index b1fd2ef3..7d456c67 100644 --- a/tests/integration/reqstool/model_generators/test_regression_structure.py +++ b/tests/integration/reqstool/model_generators/test_regression_structure.py @@ -1,5 +1,7 @@ # Copyright © reqstool +import os + import pytest from integration.reqstool.model_generators._regression_shared import ( @@ -38,7 +40,7 @@ def repo(): holder = ValidationErrorHolder() with build_database( location=GitLocation( - env_token=_GITHUB_TOKEN_ENV, + token=os.getenv(_GITHUB_TOKEN_ENV), url=_REGRESSION_REPO_URL, ref=_REGRESSION_REPO_REF, path="fixtures/parent", diff --git a/tests/resources/test_data/data/remote/test_standard/test_standard_maven_git/sys-001/requirements.yml b/tests/resources/test_data/data/remote/test_standard/test_standard_maven_git/sys-001/requirements.yml index 9b2bb33f..1fa4ab44 100644 --- a/tests/resources/test_data/data/remote/test_standard/test_standard_maven_git/sys-001/requirements.yml +++ b/tests/resources/test_data/data/remote/test_standard/test_standard_maven_git/sys-001/requirements.yml @@ -19,7 +19,7 @@ filters: implementations: git: - ref: main - env_token: GITLAB_TOKEN + token: GITLAB_TOKEN url: "https://github.com/reqstool/requirements-tool-testdata.git" path: "data/remote/test_standard/test_standard_maven_git/ms-001" diff --git a/tests/resources/unit/reqstool/model_generators/test_requirements_model_generator/test_microservice_requirements_model_generator/requirements.yml b/tests/resources/unit/reqstool/model_generators/test_requirements_model_generator/test_microservice_requirements_model_generator/requirements.yml index ab60cc7a..c368b5c9 100644 --- a/tests/resources/unit/reqstool/model_generators/test_requirements_model_generator/test_microservice_requirements_model_generator/requirements.yml +++ b/tests/resources/unit/reqstool/model_generators/test_requirements_model_generator/test_microservice_requirements_model_generator/requirements.yml @@ -8,14 +8,14 @@ metadata: imports: git: - - env_token: "GITLAB_TOKEN" + - token: "GITLAB_TOKEN" url: https://gitlab.ms-example.com ref: main path: /some/ms-path local: - path: /some/local-ms-path maven: - - env_token: "MAVEN_TOKEN" + - token: "MAVEN_TOKEN" url: https://repo.maven.org group_id: com.example.one artifact_id: test-one diff --git a/tests/resources/unit/reqstool/model_generators/test_requirements_model_generator/test_npm_requirements_model_generator/requirements.yml b/tests/resources/unit/reqstool/model_generators/test_requirements_model_generator/test_npm_requirements_model_generator/requirements.yml index dfcbdd0f..cbc5a066 100644 --- a/tests/resources/unit/reqstool/model_generators/test_requirements_model_generator/test_npm_requirements_model_generator/requirements.yml +++ b/tests/resources/unit/reqstool/model_generators/test_requirements_model_generator/test_npm_requirements_model_generator/requirements.yml @@ -7,7 +7,7 @@ metadata: imports: npm: - - env_token: "NPM_TOKEN" + - token: "NPM_TOKEN" url: https://my.registry.example.com package: "@my-org/my-service-reqstool" version: 1.2.3 diff --git a/tests/resources/unit/reqstool/model_generators/test_requirements_model_generator/test_system_requirements_model_generator/requirements.yml b/tests/resources/unit/reqstool/model_generators/test_requirements_model_generator/test_system_requirements_model_generator/requirements.yml index c35e122e..da1e53fc 100644 --- a/tests/resources/unit/reqstool/model_generators/test_requirements_model_generator/test_system_requirements_model_generator/requirements.yml +++ b/tests/resources/unit/reqstool/model_generators/test_requirements_model_generator/test_system_requirements_model_generator/requirements.yml @@ -7,7 +7,7 @@ metadata: imports: git: - - env_token: "GITLAB_TOKEN" + - token: "GITLAB_TOKEN" url: https://gitlab.sys-example.com ref: feature/sys path: /some/path @@ -15,7 +15,7 @@ imports: - path: /some/local-sys-path maven: - - env_token: "MAVEN_TOKEN" + - token: "MAVEN_TOKEN" url: https://repo.maven.org group_id: com.example.one artifact_id: test-one @@ -27,7 +27,7 @@ imports: version: 0.0.2 pypi: - - env_token: "PYPI_TOKEN" + - token: "PYPI_TOKEN" url: https://example.com/simple/ package: package1 version: 1.0.0 @@ -51,7 +51,7 @@ filters: implementations: # only used with type = system git: - - env_token: GITLAB_TOKEN + - token: GITLAB_TOKEN url: https://gitlab.impl-example.com ref: feature/impl path: README.md @@ -59,14 +59,14 @@ implementations: # only used with type = system local: # for future use - path: /some/local-impl-path maven: - - env_token: "MAVEN_TOKEN" + - token: "MAVEN_TOKEN" url: https://repo.maven.org group_id: com.example.one artifact_id: test-one version: 0.0.1 classifier: classifier1 - - env_token: "MAVEN_TOKEN" + - token: "MAVEN_TOKEN" url: https://repo2.maven.org group_id: com.example.two artifact_id: test-two diff --git a/tests/unit/reqstool/common/test_locations.py b/tests/unit/reqstool/common/test_locations.py index f43f38d8..e9f471e1 100644 --- a/tests/unit/reqstool/common/test_locations.py +++ b/tests/unit/reqstool/common/test_locations.py @@ -10,7 +10,7 @@ @pytest.fixture def git_location_data(): return GitLocation( - env_token="GITLAB_TOKEN", + token="GITLAB_TOKEN", ref="main", url="https://gitlab.example.com", path="some/path", @@ -25,7 +25,7 @@ def local_location_data(): @pytest.fixture def maven_location_data(): return MavenLocation( - env_token="MAVEN_TOKEN", + token="MAVEN_TOKEN", url="https://repo1.maven.org/maven2", group_id="com.example", artifact_id="artifactexample", @@ -35,7 +35,7 @@ def maven_location_data(): def test_git_location_data(git_location_data): - assert git_location_data.env_token == "GITLAB_TOKEN" + assert git_location_data.token == "GITLAB_TOKEN" assert git_location_data.ref == "main" assert git_location_data.url == "https://gitlab.example.com" assert git_location_data.path == "some/path" @@ -46,7 +46,7 @@ def test_local_location_data(local_location_data): def test_maven_location_data(maven_location_data): - assert maven_location_data.env_token == "MAVEN_TOKEN" + assert maven_location_data.token == "MAVEN_TOKEN" assert maven_location_data.url == "https://repo1.maven.org/maven2" assert maven_location_data.group_id == "com.example" assert maven_location_data.artifact_id == "artifactexample" diff --git a/tests/unit/reqstool/locations/test_git_location.py b/tests/unit/reqstool/locations/test_git_location.py index da0a1cdc..78490826 100644 --- a/tests/unit/reqstool/locations/test_git_location.py +++ b/tests/unit/reqstool/locations/test_git_location.py @@ -13,25 +13,25 @@ def test_git_location(): PATH = "/tmp/somepath" git_location = GitLocation( - env_token="GITLAB_TOKEN", + token="GITLAB_TOKEN", url="https://git.example.com/example/repo.git", ref="main", path=PATH, ) - assert git_location.env_token == "GITLAB_TOKEN" + assert git_location.token == "GITLAB_TOKEN" assert git_location.url == "https://git.example.com/example/repo.git" assert git_location.ref == "main" assert git_location.path == PATH git_location = GitLocation( - env_token="CI_TOKEN", + token="CI_TOKEN", url="https://git.example.com/repo.git", ref="v1.2.0", path=PATH, ) - assert git_location.env_token == "CI_TOKEN" + assert git_location.token == "CI_TOKEN" assert git_location.url == "https://git.example.com/repo.git" assert git_location.ref == "v1.2.0" assert git_location.path == PATH @@ -54,13 +54,13 @@ def test_git_location_explicit_path_still_works(): assert git_location.path == "docs/reqstool" -def test_git_location_env_token_defaults_to_none(): +def test_git_location_token_defaults_to_none(): git_location = GitLocation( url="https://git.example.com/repo.git", ref="main", path="/tmp/somepath", ) - assert git_location.env_token is None + assert git_location.token is None @pytest.mark.parametrize( diff --git a/tests/unit/reqstool/locations/test_maven_location.py b/tests/unit/reqstool/locations/test_maven_location.py index 2f5ec09b..caad1259 100644 --- a/tests/unit/reqstool/locations/test_maven_location.py +++ b/tests/unit/reqstool/locations/test_maven_location.py @@ -5,12 +5,12 @@ from reqstool.locations.maven_location import MavenLocation -def test_maven_location_env_token_defaults_to_none(): +def test_maven_location_token_defaults_to_none(): loc = MavenLocation(group_id="com.example", artifact_id="my-lib", version="1.0.0") - assert loc.env_token is None + assert loc.token is None -def test_maven_location_make_available_no_env_token(tmp_path): +def test_maven_location_make_available_no_token(tmp_path): loc = MavenLocation(group_id="com.example", artifact_id="my-lib", version="1.0.0") mock_downloader = MagicMock() mock_downloader.download.return_value = True diff --git a/tests/unit/reqstool/locations/test_npm_location.py b/tests/unit/reqstool/locations/test_npm_location.py index 9b4607f7..437b6310 100644 --- a/tests/unit/reqstool/locations/test_npm_location.py +++ b/tests/unit/reqstool/locations/test_npm_location.py @@ -11,7 +11,7 @@ def test_npm_location_defaults(): loc = NpmLocation(package="@scope/my-pkg-reqstool", version="1.2.3") assert loc.url == "https://registry.npmjs.org" - assert loc.env_token is None + assert loc.token is None def test_npm_location_custom_registry(): @@ -37,9 +37,8 @@ def test_npm_location_make_available_no_token(tmp_path): assert result == extracted -def test_npm_location_make_available_with_token(tmp_path, monkeypatch): - monkeypatch.setenv("NPM_TOKEN", "secret-token") - loc = NpmLocation(package="my-pkg-reqstool", version="2.0.0", env_token="NPM_TOKEN") +def test_npm_location_make_available_with_token(tmp_path): + loc = NpmLocation(package="my-pkg-reqstool", version="2.0.0", token="secret-token") mock_response = MagicMock() mock_response.json.return_value = {"dist": {"tarball": "https://registry.npmjs.org/tarball.tgz"}} @@ -205,9 +204,8 @@ def test_npm_location_get_tarball_uses_timeout(tmp_path): assert mock_get.call_args[1].get("timeout") == 30 -def test_npm_location_make_available_token_env_var_absent_sends_no_auth(tmp_path, monkeypatch): - monkeypatch.delenv("NPM_TOKEN", raising=False) - loc = NpmLocation(package="my-pkg-reqstool", version="1.0.0", env_token="NPM_TOKEN") +def test_npm_location_make_available_no_token_sends_no_auth(tmp_path): + loc = NpmLocation(package="my-pkg-reqstool", version="1.0.0") mock_response = MagicMock() mock_response.json.return_value = {"dist": {"tarball": "https://registry.npmjs.org/tarball.tgz"}} diff --git a/tests/unit/reqstool/locations/test_pypi_location.py b/tests/unit/reqstool/locations/test_pypi_location.py index ff85f972..63445749 100644 --- a/tests/unit/reqstool/locations/test_pypi_location.py +++ b/tests/unit/reqstool/locations/test_pypi_location.py @@ -5,12 +5,12 @@ from reqstool.locations.pypi_location import PypiLocation -def test_pypi_location_env_token_defaults_to_none(): +def test_pypi_location_token_defaults_to_none(): loc = PypiLocation(package="my-package", version="1.0.0") - assert loc.env_token is None + assert loc.token is None -def test_pypi_location_make_available_no_env_token(tmp_path): +def test_pypi_location_make_available_no_token(tmp_path): loc = PypiLocation(package="my-package", version="1.0.0") extracted = str(tmp_path / "extracted") with ( diff --git a/tests/unit/reqstool/model_generators/test_requirements_model_generator.py b/tests/unit/reqstool/model_generators/test_requirements_model_generator.py index aaf8445b..22e7e74e 100644 --- a/tests/unit/reqstool/model_generators/test_requirements_model_generator.py +++ b/tests/unit/reqstool/model_generators/test_requirements_model_generator.py @@ -45,13 +45,13 @@ def test_system_requirements_model_generator(resource_funcname_rootdir_w_path): assert model.imports[0].current_unresolved.path == "/some/local-sys-path" # git - assert model.imports[1].current_unresolved.env_token == "GITLAB_TOKEN" + assert model.imports[1].current_unresolved.token == "GITLAB_TOKEN" assert model.imports[1].current_unresolved.url == "https://gitlab.sys-example.com" assert model.imports[1].current_unresolved.ref == "feature/sys" assert model.imports[1].current_unresolved.path == "/some/path" # maven #1 - assert model.imports[2].current_unresolved.env_token == "MAVEN_TOKEN" + assert model.imports[2].current_unresolved.token == "MAVEN_TOKEN" assert model.imports[2].current_unresolved.url == "https://repo.maven.org" assert model.imports[2].current_unresolved.group_id == "com.example.one" assert model.imports[2].current_unresolved.artifact_id == "test-one" @@ -59,7 +59,7 @@ def test_system_requirements_model_generator(resource_funcname_rootdir_w_path): assert model.imports[2].current_unresolved.classifier == "someclassifier" # maven #2 - assert model.imports[3].current_unresolved.env_token is None + assert model.imports[3].current_unresolved.token is None assert model.imports[3].current_unresolved.url == "https://repo.maven.apache.org/maven2/" assert model.imports[3].current_unresolved.group_id == "com.example.two" assert model.imports[3].current_unresolved.artifact_id == "test-two" @@ -67,13 +67,13 @@ def test_system_requirements_model_generator(resource_funcname_rootdir_w_path): assert model.imports[3].current_unresolved.classifier == "reqstool" # pypi #1 - assert model.imports[4].current_unresolved.env_token == "PYPI_TOKEN" + assert model.imports[4].current_unresolved.token == "PYPI_TOKEN" assert model.imports[4].current_unresolved.url == "https://example.com/simple/" assert model.imports[4].current_unresolved.package == "package1" assert model.imports[4].current_unresolved.version == "1.0.0" # pypi #2 - assert model.imports[5].current_unresolved.env_token is None + assert model.imports[5].current_unresolved.token is None assert model.imports[5].current_unresolved.url == "https://pypi.org/simple/" assert model.imports[5].current_unresolved.package == "package2" assert model.imports[5].current_unresolved.version == "2.0.0" @@ -85,7 +85,7 @@ def test_system_requirements_model_generator(resource_funcname_rootdir_w_path): assert model.implementations[0].current_unresolved.path == "/some/local-impl-path" # git - assert model.implementations[1].current_unresolved.env_token == "GITLAB_TOKEN" + assert model.implementations[1].current_unresolved.token == "GITLAB_TOKEN" assert model.implementations[1].current_unresolved.url == "https://gitlab.impl-example.com" assert model.implementations[1].current_unresolved.ref == "feature/impl" assert model.implementations[1].current_unresolved.path == "README.md" @@ -147,7 +147,7 @@ def test_microservice_requirements_model_generator(resource_funcname_rootdir_w_p assert model.imports[0].current_unresolved.path == "/some/local-ms-path" # git - assert model.imports[1].current_unresolved.env_token == "GITLAB_TOKEN" + assert model.imports[1].current_unresolved.token == "GITLAB_TOKEN" assert model.imports[1].current_unresolved.url == "https://gitlab.ms-example.com" assert model.imports[1].current_unresolved.ref == "main" assert model.imports[1].current_unresolved.path == "/some/ms-path" @@ -278,7 +278,7 @@ def test_npm_requirements_model_generator(resource_funcname_rootdir_w_path): # npm import #1 — with token and custom registry npm_import_1 = model.imports[0].current_unresolved assert isinstance(npm_import_1, NpmLocation) - assert npm_import_1.env_token == "NPM_TOKEN" + assert npm_import_1.token == "NPM_TOKEN" assert str(npm_import_1.url) == "https://my.registry.example.com" assert npm_import_1.package == "@my-org/my-service-reqstool" assert npm_import_1.version == "1.2.3" @@ -286,7 +286,7 @@ def test_npm_requirements_model_generator(resource_funcname_rootdir_w_path): # npm import #2 — default registry, no token npm_import_2 = model.imports[1].current_unresolved assert isinstance(npm_import_2, NpmLocation) - assert npm_import_2.env_token is None + assert npm_import_2.token is None assert npm_import_2.url == "https://registry.npmjs.org" assert npm_import_2.package == "my-lib-reqstool" assert npm_import_2.version == "0.5.0" diff --git a/tests/unit/reqstool/models/test_implementations.py b/tests/unit/reqstool/models/test_implementations.py index 3eb9b0e8..a0dbd6b6 100644 --- a/tests/unit/reqstool/models/test_implementations.py +++ b/tests/unit/reqstool/models/test_implementations.py @@ -13,7 +13,7 @@ def git_impl_data() -> GitImplData: return GitImplData( parent=None, current_unresolved=GitLocation( - env_token="GITLAB_TOKEN", + token="GITLAB_TOKEN", ref="main", url="https://github.com/reqstool/reqstool-client", path="/examples/README.adoc", @@ -31,7 +31,7 @@ def maven_impl_data() -> MavenImplData: return MavenImplData( parent=None, current_unresolved=MavenLocation( - env_token="MAVEN_TOKEN", + token="MAVEN_TOKEN", url="https://repo1.maven.org/maven2", group_id="com.example", artifact_id="artifactexample", @@ -48,7 +48,7 @@ def implementations_data(maven_impl_data): def test_git_impl_data(git_impl_data): assert git_impl_data.parent is None - assert git_impl_data.current.env_token == "GITLAB_TOKEN" + assert git_impl_data.current.token == "GITLAB_TOKEN" assert git_impl_data.current.ref == "main" assert git_impl_data.current.url == "https://github.com/reqstool/reqstool-client" assert git_impl_data.current.path == "/examples/README.adoc" @@ -61,7 +61,7 @@ def test_local_impl_data(local_impl_data): def test_maven_impl_data(maven_impl_data): assert maven_impl_data.parent is None - assert maven_impl_data.current.env_token == "MAVEN_TOKEN" + assert maven_impl_data.current.token == "MAVEN_TOKEN" assert maven_impl_data.current.url == "https://repo1.maven.org/maven2" assert maven_impl_data.current.group_id == "com.example" assert maven_impl_data.current.artifact_id == "artifactexample" diff --git a/tests/unit/reqstool/models/test_imports.py b/tests/unit/reqstool/models/test_imports.py index 1bbc7cb9..d8193a0a 100644 --- a/tests/unit/reqstool/models/test_imports.py +++ b/tests/unit/reqstool/models/test_imports.py @@ -13,7 +13,7 @@ def git_import_data(): return imports.GitImportData( parent=None, current_unresolved=GitLocation( - env_token="GITLAB_TOKEN", + token="GITLAB_TOKEN", ref="main", url="https://gitlab.example.com", path="git/some/path", @@ -31,7 +31,7 @@ def maven_import_data(): return imports.MavenImportData( parent=None, current_unresolved=MavenLocation( - env_token="MAVEN_TOKEN", + token="MAVEN_TOKEN", url="https://repo1.maven.org/maven2", group_id="com.example", artifact_id="artifactexample", @@ -43,7 +43,7 @@ def maven_import_data(): def test_git_import_data(git_import_data): assert git_import_data.parent is None - assert git_import_data.current.env_token == "GITLAB_TOKEN" + assert git_import_data.current.token == "GITLAB_TOKEN" assert git_import_data.current.ref == "main" assert git_import_data.current.url == "https://gitlab.example.com" assert git_import_data.current.path == "git/some/path" @@ -56,7 +56,7 @@ def test_local_system_data(local_import_data): def test_maven_system_data(maven_import_data): assert maven_import_data.parent is None - assert maven_import_data.current.env_token == "MAVEN_TOKEN" + assert maven_import_data.current.token == "MAVEN_TOKEN" assert maven_import_data.current.url == "https://repo1.maven.org/maven2" assert maven_import_data.current.group_id == "com.example" assert maven_import_data.current.artifact_id == "artifactexample" diff --git a/tests/unit/reqstool/test_command.py b/tests/unit/reqstool/test_command.py index 45ba9717..1986225c 100644 --- a/tests/unit/reqstool/test_command.py +++ b/tests/unit/reqstool/test_command.py @@ -196,12 +196,12 @@ def test_npm_source_parser_accepts_custom_url_and_token(): "1.0.0", "--url", "https://my.registry.example.com", - "--env_token", + "--token", "NPM_TOKEN", ] ) assert args.url == "https://my.registry.example.com" - assert args.env_token == "NPM_TOKEN" + assert args.token == "NPM_TOKEN" def test_local_source_parser_accepts_npm_path_arg(): @@ -211,7 +211,7 @@ def test_local_source_parser_accepts_npm_path_arg(): def test_get_initial_source_npm_returns_npm_location(): - args = argparse.Namespace(source="npm", package="my-pkg-reqstool", version="1.0.0", url=None, env_token=None) + args = argparse.Namespace(source="npm", package="my-pkg-reqstool", version="1.0.0", url=None, token=None) loc = Command()._get_initial_source(args) assert isinstance(loc, NpmLocation) assert loc.package == "my-pkg-reqstool" @@ -225,12 +225,12 @@ def test_get_initial_source_npm_with_custom_url(): package="my-pkg-reqstool", version="1.0.0", url="https://my.registry.example.com", - env_token="NPM_TOKEN", + token="NPM_TOKEN", ) loc = Command()._get_initial_source(args) assert isinstance(loc, NpmLocation) assert loc.url == "https://my.registry.example.com" - assert loc.env_token == "NPM_TOKEN" + assert loc.token == "NPM_TOKEN" def test_get_initial_source_local_npm_returns_local_npm_location(): From 838f6d4bc65995596c0f4ec6a4488fa9d184a8ea Mon Sep 17 00:00:00 2001 From: Jimisola Laursen Date: Mon, 1 Jun 2026 18:08:01 +0200 Subject: [PATCH 2/5] style: apply black formatting to command.py Signed-off-by: Jimisola Laursen --- src/reqstool/command.py | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/src/reqstool/command.py b/src/reqstool/command.py index f5953b01..b7a42e45 100755 --- a/src/reqstool/command.py +++ b/src/reqstool/command.py @@ -74,7 +74,10 @@ {"flags": ["-u", "--url"], "kwargs": {"help": "git repository URL", "required": True}}, {"flags": ["-p", "--path"], "kwargs": {"help": "path within the repository", "required": True}}, {"flags": ["-r", "--ref"], "kwargs": {"help": "git branch, tag, or commit SHA", "required": True}}, - {"flags": ["-t", "--token"], "kwargs": {"help": "authentication token value (use $VAR to read from environment)"}}, + { + "flags": ["-t", "--token"], + "kwargs": {"help": "authentication token value (use $VAR to read from environment)"}, + }, ], }, { @@ -82,7 +85,10 @@ "help": "maven source", "args": [ {"flags": ["-u", "--url"], "kwargs": {"help": "Maven repository URL", "required": False}}, - {"flags": ["-t", "--token"], "kwargs": {"help": "authentication token value (use $VAR to read from environment)"}}, + { + "flags": ["-t", "--token"], + "kwargs": {"help": "authentication token value (use $VAR to read from environment)"}, + }, {"flags": ["--group_id"], "kwargs": {"help": "Maven group ID", "required": True}}, {"flags": ["--artifact_id"], "kwargs": {"help": "Maven artifact ID", "required": True}}, {"flags": ["--version"], "kwargs": {"help": "artifact version (e.g. 1.2.3)", "required": True}}, @@ -100,7 +106,10 @@ "required": False, }, }, - {"flags": ["-t", "--token"], "kwargs": {"help": "authentication token value (use $VAR to read from environment)"}}, + { + "flags": ["-t", "--token"], + "kwargs": {"help": "authentication token value (use $VAR to read from environment)"}, + }, {"flags": ["--package"], "kwargs": {"help": "npm package name (e.g. @scope/package)", "required": True}}, {"flags": ["--version"], "kwargs": {"help": "package version (e.g. 1.2.3)", "required": True}}, ], @@ -110,7 +119,10 @@ "help": "pypi source", "args": [ {"flags": ["-u", "--url"], "kwargs": {"help": "PyPI index URL", "required": False}}, - {"flags": ["-t", "--token"], "kwargs": {"help": "authentication token value (use $VAR to read from environment)"}}, + { + "flags": ["-t", "--token"], + "kwargs": {"help": "authentication token value (use $VAR to read from environment)"}, + }, {"flags": ["--package"], "kwargs": {"help": "PyPI package name", "required": True}}, {"flags": ["--version"], "kwargs": {"help": "package version (e.g. 1.2.3)", "required": True}}, ], From 0956a07da0a39c8ad7be35222bba3fa61af7a6e1 Mon Sep 17 00:00:00 2001 From: Jimisola Laursen Date: Mon, 1 Jun 2026 18:46:46 +0200 Subject: [PATCH 3/5] refactor(locations): address full-pr-review findings MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - SecretStr: use pydantic.SecretStr for token field in all 4 location classes — prevents raw secret from appearing in repr/model_dump/ ValidationError messages; callers extract value via .get_secret_value() - Credentials: pass callbacks=None to clone_repository when no token, eliminating UserPass(password=None) passed to pygit2 - Schema: add writeOnly:true to all 4 token property definitions and regenerate models - Docs examples: quote token interpolation as token: "${VAR}" to prevent YAML injection when env value contains structural characters - Remote fixture: change literal token to token: "${GITLAB_TOKEN:-}" - CLI help: fix $VAR → ${VAR} in all --token help strings - CLI ternaries: simplify x if x else None → x or None (×4) - Integration tests: fix choose_token() skipif; rename _GITHUB_TOKEN_ENV → _GITHUB_TOKEN_VAR_NAME - Test fixtures: use clearly dummy token values (dummy-gitlab-token etc.) - Exception message: use type(e).__name__ instead of {e} in pypi - Copyright: add missing # Copyright © LFV header to pypi_location.py - New tests: token forwarding for git/maven/pypi; empty-string no-auth; repr does not expose secret value; --token CLI flag for git/maven/pypi Signed-off-by: Jimisola Laursen --- .gitignore | 1 + .../requirements_microservice.yml | 6 +- .../requirements/requirements_system.yml | 4 +- src/reqstool/command.py | 16 +++--- src/reqstool/locations/git_location.py | 9 +-- src/reqstool/locations/maven_location.py | 5 +- src/reqstool/locations/npm_location.py | 6 +- src/reqstool/locations/pypi_location.py | 11 ++-- .../schemas/v1/requirements.schema.json | 4 ++ .../model_generators/_regression_shared.py | 2 +- .../reqstool/model_generators/conftest.py | 6 +- .../test_included_models_generator.py | 8 +-- .../test_regression_monorepo.py | 4 +- .../test_regression_structure.py | 4 +- .../sys-001/requirements.yml | 2 +- .../requirements.yml | 4 +- .../requirements.yml | 2 +- .../requirements.yml | 12 ++-- tests/unit/reqstool/common/test_locations.py | 4 +- .../reqstool/locations/test_git_location.py | 31 +++++++++- .../reqstool/locations/test_maven_location.py | 21 ++++++- .../reqstool/locations/test_npm_location.py | 19 +++++++ .../reqstool/locations/test_pypi_location.py | 32 ++++++++++- .../test_requirements_model_generator.py | 12 ++-- .../reqstool/models/test_implementations.py | 4 +- tests/unit/reqstool/models/test_imports.py | 4 +- tests/unit/reqstool/test_command.py | 57 ++++++++++++++++++- 27 files changed, 225 insertions(+), 65 deletions(-) diff --git a/.gitignore b/.gitignore index f0277f49..52b3d38f 100644 --- a/.gitignore +++ b/.gitignore @@ -284,3 +284,4 @@ pyrightconfig.json # Claude Code .claude/ +workspace/ diff --git a/docs/modules/examples/partials/requirements/requirements_microservice.yml b/docs/modules/examples/partials/requirements/requirements_microservice.yml index d8b47503..e0eba243 100644 --- a/docs/modules/examples/partials/requirements/requirements_microservice.yml +++ b/docs/modules/examples/partials/requirements/requirements_microservice.yml @@ -8,19 +8,19 @@ metadata: imports: #Optional git: - - token: ${GITLAB_TOKEN} # use ${VAR} interpolation to supply from environment + - token: "${GITLAB_TOKEN}" # use ${VAR} interpolation to supply from environment ref: url: url path: path maven: - - token: ${MAVEN_TOKEN} # use ${VAR} interpolation to supply from environment + - token: "${MAVEN_TOKEN}" # use ${VAR} interpolation to supply from environment url: x.se group_id: x artifact_id: y classifier: s version: 1.0.0 pypi: - - token: ${PYPI_TOKEN} # use ${VAR} interpolation to supply from environment + - token: "${PYPI_TOKEN}" # use ${VAR} interpolation to supply from environment url: url package: package-name version: 1.0.0 diff --git a/docs/modules/examples/partials/requirements/requirements_system.yml b/docs/modules/examples/partials/requirements/requirements_system.yml index 9cdf5cb7..cf94b9f7 100644 --- a/docs/modules/examples/partials/requirements/requirements_system.yml +++ b/docs/modules/examples/partials/requirements/requirements_system.yml @@ -8,7 +8,7 @@ metadata: imports: git: - - token: ${GITLAB_TOKEN} # use ${VAR} interpolation to supply from environment + - token: "${GITLAB_TOKEN}" # use ${VAR} interpolation to supply from environment ref: url: path: @@ -26,7 +26,7 @@ filters: implementations: # only used with type = system git: - - token: ${GITLAB_TOKEN} # use ${VAR} interpolation to supply from environment + - token: "${GITLAB_TOKEN}" # use ${VAR} interpolation to supply from environment ref: url: path: diff --git a/src/reqstool/command.py b/src/reqstool/command.py index b7a42e45..1f2d45d3 100755 --- a/src/reqstool/command.py +++ b/src/reqstool/command.py @@ -76,7 +76,7 @@ {"flags": ["-r", "--ref"], "kwargs": {"help": "git branch, tag, or commit SHA", "required": True}}, { "flags": ["-t", "--token"], - "kwargs": {"help": "authentication token value (use $VAR to read from environment)"}, + "kwargs": {"help": "authentication token value (use ${VAR} in YAML or shell expansion for CLI)"}, }, ], }, @@ -87,7 +87,7 @@ {"flags": ["-u", "--url"], "kwargs": {"help": "Maven repository URL", "required": False}}, { "flags": ["-t", "--token"], - "kwargs": {"help": "authentication token value (use $VAR to read from environment)"}, + "kwargs": {"help": "authentication token value (use ${VAR} in YAML or shell expansion for CLI)"}, }, {"flags": ["--group_id"], "kwargs": {"help": "Maven group ID", "required": True}}, {"flags": ["--artifact_id"], "kwargs": {"help": "Maven artifact ID", "required": True}}, @@ -108,7 +108,7 @@ }, { "flags": ["-t", "--token"], - "kwargs": {"help": "authentication token value (use $VAR to read from environment)"}, + "kwargs": {"help": "authentication token value (use ${VAR} in YAML or shell expansion for CLI)"}, }, {"flags": ["--package"], "kwargs": {"help": "npm package name (e.g. @scope/package)", "required": True}}, {"flags": ["--version"], "kwargs": {"help": "package version (e.g. 1.2.3)", "required": True}}, @@ -121,7 +121,7 @@ {"flags": ["-u", "--url"], "kwargs": {"help": "PyPI index URL", "required": False}}, { "flags": ["-t", "--token"], - "kwargs": {"help": "authentication token value (use $VAR to read from environment)"}, + "kwargs": {"help": "authentication token value (use ${VAR} in YAML or shell expansion for CLI)"}, }, {"flags": ["--package"], "kwargs": {"help": "PyPI package name", "required": True}}, {"flags": ["--version"], "kwargs": {"help": "package version (e.g. 1.2.3)", "required": True}}, @@ -449,28 +449,28 @@ def _get_initial_source(self, args_source: argparse.Namespace) -> LocationInterf artifact_id=args_source.artifact_id, version=args_source.version, classifier=args_source.classifier if args_source.classifier else None, - token=args_source.token if args_source.token else None, + token=args_source.token or None, ) elif args_source.source == "npm": location = NpmLocation( url=args_source.url if args_source.url else "https://registry.npmjs.org", package=args_source.package, version=args_source.version, - token=args_source.token if args_source.token else None, + token=args_source.token or None, ) elif args_source.source == "pypi": location = PypiLocation( url=args_source.url if args_source.url else None, package=args_source.package, version=args_source.version, - token=args_source.token if args_source.token else None, + token=args_source.token or None, ) elif args_source.source == "git": location = GitLocation( url=args_source.url, path=args_source.path, ref=args_source.ref, - token=args_source.token if args_source.token else None, + token=args_source.token or None, ) elif args_source.source == "local": if args_source.maven: diff --git a/src/reqstool/locations/git_location.py b/src/reqstool/locations/git_location.py index bd067dcf..5c2830dc 100644 --- a/src/reqstool/locations/git_location.py +++ b/src/reqstool/locations/git_location.py @@ -4,7 +4,7 @@ import re from typing import Optional -from pydantic import field_validator +from pydantic import SecretStr, field_validator from pygit2 import Commit, GitError, RemoteCallbacks, UserPass, clone_repository from reqstool_python_decorators.decorators.decorators import Requirements @@ -20,7 +20,7 @@ class GitLocation(LocationInterface): url: str ref: str - token: Optional[str] = None + token: Optional[SecretStr] = None path: str = "" @field_validator("ref") @@ -43,9 +43,10 @@ def tmpdir_key(self) -> str: return make_safe_tmpdir_suffix("git", f"{urlunparse(parsed)}@{self.ref}") def _make_available_on_localdisk(self, dst_path: str) -> str: - api_token = self.token + api_token = self.token.get_secret_value() if self.token else None + callbacks = self.MyRemoteCallbacks(api_token) if api_token else None - repo = clone_repository(url=self.url, path=dst_path, callbacks=self.MyRemoteCallbacks(api_token)) + repo = clone_repository(url=self.url, path=dst_path, callbacks=callbacks) # Try direct lookup first (tag, default branch, or commit SHA), then fall back to the # remote-tracking ref — non-default branches only exist as origin/ after a plain clone. diff --git a/src/reqstool/locations/maven_location.py b/src/reqstool/locations/maven_location.py index 30beaea3..da7d67dd 100644 --- a/src/reqstool/locations/maven_location.py +++ b/src/reqstool/locations/maven_location.py @@ -4,6 +4,7 @@ from typing import Optional from maven_artifact import Artifact, Downloader, RequestException +from pydantic import SecretStr from reqstool_python_decorators.decorators.decorators import Requirements from reqstool.common.exceptions import ArtifactDownloadError, ArtifactExtractionError @@ -18,13 +19,13 @@ class MavenLocation(LocationInterface): artifact_id: str version: str classifier: str = "reqstool" - token: Optional[str] = None + token: Optional[SecretStr] = None def tmpdir_key(self) -> str: return make_safe_tmpdir_suffix("maven", f"{self.group_id}:{self.artifact_id}:{self.version}") def _make_available_on_localdisk(self, dst_path: str): - token = self.token + token = self.token.get_secret_value() if self.token else None # assume OAuth Bearer, see: https://georgearisty.dev/posts/oauth2-token-bearer-usage/ downloader = Downloader(base=self.url, token=token) diff --git a/src/reqstool/locations/npm_location.py b/src/reqstool/locations/npm_location.py index d2c343ea..39dcd09f 100644 --- a/src/reqstool/locations/npm_location.py +++ b/src/reqstool/locations/npm_location.py @@ -6,7 +6,7 @@ from urllib.parse import quote, urlparse import requests -from pydantic import field_validator +from pydantic import SecretStr, field_validator from reqstool.common.exceptions import ArtifactDownloadError, ArtifactExtractionError from reqstool.common.utils import Utils @@ -20,7 +20,7 @@ class NpmLocation(LocationInterface): url: str = "https://registry.npmjs.org" package: str version: str - token: Optional[str] = None + token: Optional[SecretStr] = None @field_validator("url") @classmethod @@ -34,7 +34,7 @@ def tmpdir_key(self) -> str: def _make_available_on_localdisk(self, dst_path: str): """Fetch tarball URL → SSRF check → download → extract.""" - token = self.token + token = self.token.get_secret_value() if self.token else None try: tarball_url = self._get_tarball_url(token) diff --git a/src/reqstool/locations/pypi_location.py b/src/reqstool/locations/pypi_location.py index 4f51e146..036ef37c 100644 --- a/src/reqstool/locations/pypi_location.py +++ b/src/reqstool/locations/pypi_location.py @@ -1,3 +1,5 @@ +# Copyright © LFV + import logging import re import tarfile @@ -6,6 +8,7 @@ import requests from bs4 import BeautifulSoup +from pydantic import SecretStr from reqstool.common.exceptions import ArtifactDownloadError, ArtifactExtractionError from reqstool.common.utils import Utils from reqstool.locations.location import LocationInterface, make_safe_tmpdir_suffix @@ -15,7 +18,7 @@ class PypiLocation(LocationInterface): url: str = "https://pypi.org/simple" package: str version: str - token: Optional[str] = None + token: Optional[SecretStr] = None @staticmethod def normalize_pypi_package_name(package_name): @@ -25,7 +28,7 @@ def tmpdir_key(self) -> str: return make_safe_tmpdir_suffix("pypi", f"{self.package}=={self.version}") def _make_available_on_localdisk(self, dst_path: str): - token = self.token + token = self.token.get_secret_value() if self.token else None if token: logging.debug("Using OAuth Bearer token for authentication") @@ -33,7 +36,7 @@ def _make_available_on_localdisk(self, dst_path: str): package_url = self.get_package_url(self.package, self.version, self.url, token) if not package_url: - token_info = "(with authentication token)" if self.token else "" + token_info = "(with authentication token)" if token else "" raise RuntimeError( f"Unable to find a sdist pypi package for {self.package} == {self.version} in repo {self.url}{token_info}" ) @@ -49,7 +52,7 @@ def _make_available_on_localdisk(self, dst_path: str): except Exception as e: raise ArtifactDownloadError( f"Error when downloading etc sdist pypi package for {self.package}=={self.version}" - f" in repo {self.url} {'with token' if token else ''}: {e}" + f" in repo {self.url} {'with token' if token else ''}: {type(e).__name__}" ) from e @staticmethod diff --git a/src/reqstool/resources/schemas/v1/requirements.schema.json b/src/reqstool/resources/schemas/v1/requirements.schema.json index ded707fd..97c34b71 100644 --- a/src/reqstool/resources/schemas/v1/requirements.schema.json +++ b/src/reqstool/resources/schemas/v1/requirements.schema.json @@ -111,6 +111,7 @@ "properties": { "token": { "type": "string", + "writeOnly": true, "description": "Authentication token value. Use ${VAR} interpolation to supply from environment, e.g. ${GITLAB_TOKEN}." }, "ref": { @@ -137,6 +138,7 @@ "properties": { "token": { "type": "string", + "writeOnly": true, "description": "Authentication token value. Use ${VAR} interpolation to supply from environment, e.g. ${MAVEN_TOKEN}." }, "url": { @@ -173,6 +175,7 @@ "properties": { "token": { "type": "string", + "writeOnly": true, "description": "Authentication token value. Use ${VAR} interpolation to supply from environment, e.g. ${NPM_TOKEN}." }, "url": { @@ -200,6 +203,7 @@ "properties": { "token": { "type": "string", + "writeOnly": true, "description": "Authentication token value. Use ${VAR} interpolation to supply from environment, e.g. ${PYPI_TOKEN}." }, "url": { diff --git a/tests/integration/reqstool/model_generators/_regression_shared.py b/tests/integration/reqstool/model_generators/_regression_shared.py index ae2768bd..ba9b8026 100644 --- a/tests/integration/reqstool/model_generators/_regression_shared.py +++ b/tests/integration/reqstool/model_generators/_regression_shared.py @@ -8,7 +8,7 @@ # _REGRESSION_REPO_REF = "v1.0.0" # resolves directly (no fallback needed) # _REGRESSION_REPO_REF = "abc1234" # resolves directly as commit SHA _REGRESSION_REPO_REF = "main" -_GITHUB_TOKEN_ENV = "GITHUB_TOKEN" +_GITHUB_TOKEN_VAR_NAME = "GITHUB_TOKEN" # Single source of truth for ecosystem names. # Both ECOSYSTEM_PATHS (for entry-point tests) and ECOSYSTEM_URNS (for structural tests) diff --git a/tests/integration/reqstool/model_generators/conftest.py b/tests/integration/reqstool/model_generators/conftest.py index d714b58f..2c7b120d 100644 --- a/tests/integration/reqstool/model_generators/conftest.py +++ b/tests/integration/reqstool/model_generators/conftest.py @@ -4,14 +4,14 @@ import pytest -from integration.reqstool.model_generators._regression_shared import _GITHUB_TOKEN_ENV +from integration.reqstool.model_generators._regression_shared import _GITHUB_TOKEN_VAR_NAME from reqstool.common.utils import TempDirectoryManager pytestmark = [ pytest.mark.integration, pytest.mark.skipif( - not os.getenv(_GITHUB_TOKEN_ENV, "").strip(), - reason=f"Test needs {_GITHUB_TOKEN_ENV}", + not os.getenv(_GITHUB_TOKEN_VAR_NAME, "").strip(), + reason=f"Test needs {_GITHUB_TOKEN_VAR_NAME}", ), ] diff --git a/tests/integration/reqstool/model_generators/test_included_models_generator.py b/tests/integration/reqstool/model_generators/test_included_models_generator.py index f375b852..c34b89ec 100644 --- a/tests/integration/reqstool/model_generators/test_included_models_generator.py +++ b/tests/integration/reqstool/model_generators/test_included_models_generator.py @@ -19,8 +19,8 @@ def choose_token(): @SVCs("SVC_002") @pytest.mark.integration @pytest.mark.skipif( - not (os.getenv("GITHUB_TOKEN")), - reason="Test needs GITHUB_TOKEN", + not (os.getenv("GITHUB_TOKEN") or os.getenv("GITLAB_TOKEN")), + reason="Test needs GITHUB_TOKEN or GITLAB_TOKEN", ) def test_basic_git(): semantic_validator = SemanticValidator(validation_error_holder=ValidationErrorHolder()) @@ -39,8 +39,8 @@ def test_basic_git(): @SVCs("SVC_003", "SVC_008") @pytest.mark.integration @pytest.mark.skipif( - not (os.getenv("GITHUB_TOKEN")), - reason="Test needs GITHUB_TOKEN", + not (os.getenv("GITHUB_TOKEN") or os.getenv("GITLAB_TOKEN")), + reason="Test needs GITHUB_TOKEN or GITLAB_TOKEN", ) def test_basic_maven(): semantic_validator = SemanticValidator(validation_error_holder=ValidationErrorHolder()) diff --git a/tests/integration/reqstool/model_generators/test_regression_monorepo.py b/tests/integration/reqstool/model_generators/test_regression_monorepo.py index 871c87f6..56db3243 100644 --- a/tests/integration/reqstool/model_generators/test_regression_monorepo.py +++ b/tests/integration/reqstool/model_generators/test_regression_monorepo.py @@ -6,7 +6,7 @@ from integration.reqstool.model_generators._regression_shared import ( ECOSYSTEM_PATHS, - _GITHUB_TOKEN_ENV, + _GITHUB_TOKEN_VAR_NAME, _REGRESSION_REPO_REF, _REGRESSION_REPO_URL, ) @@ -30,7 +30,7 @@ def _make_generator(path: str, tmpdir_manager=None): holder = ValidationErrorHolder() gen = combined_raw_datasets_generator.CombinedRawDatasetsGenerator( initial_location=GitLocation( - token=os.getenv(_GITHUB_TOKEN_ENV), + token=os.getenv(_GITHUB_TOKEN_VAR_NAME), url=_REGRESSION_REPO_URL, ref=_REGRESSION_REPO_REF, path=path, diff --git a/tests/integration/reqstool/model_generators/test_regression_structure.py b/tests/integration/reqstool/model_generators/test_regression_structure.py index 7d456c67..a9676b69 100644 --- a/tests/integration/reqstool/model_generators/test_regression_structure.py +++ b/tests/integration/reqstool/model_generators/test_regression_structure.py @@ -6,7 +6,7 @@ from integration.reqstool.model_generators._regression_shared import ( ECOSYSTEM_URNS, - _GITHUB_TOKEN_ENV, + _GITHUB_TOKEN_VAR_NAME, _REGRESSION_REPO_REF, _REGRESSION_REPO_URL, ) @@ -40,7 +40,7 @@ def repo(): holder = ValidationErrorHolder() with build_database( location=GitLocation( - token=os.getenv(_GITHUB_TOKEN_ENV), + token=os.getenv(_GITHUB_TOKEN_VAR_NAME), url=_REGRESSION_REPO_URL, ref=_REGRESSION_REPO_REF, path="fixtures/parent", diff --git a/tests/resources/test_data/data/remote/test_standard/test_standard_maven_git/sys-001/requirements.yml b/tests/resources/test_data/data/remote/test_standard/test_standard_maven_git/sys-001/requirements.yml index 1fa4ab44..be543299 100644 --- a/tests/resources/test_data/data/remote/test_standard/test_standard_maven_git/sys-001/requirements.yml +++ b/tests/resources/test_data/data/remote/test_standard/test_standard_maven_git/sys-001/requirements.yml @@ -19,7 +19,7 @@ filters: implementations: git: - ref: main - token: GITLAB_TOKEN + token: "${GITLAB_TOKEN:-}" url: "https://github.com/reqstool/requirements-tool-testdata.git" path: "data/remote/test_standard/test_standard_maven_git/ms-001" diff --git a/tests/resources/unit/reqstool/model_generators/test_requirements_model_generator/test_microservice_requirements_model_generator/requirements.yml b/tests/resources/unit/reqstool/model_generators/test_requirements_model_generator/test_microservice_requirements_model_generator/requirements.yml index c368b5c9..f805122f 100644 --- a/tests/resources/unit/reqstool/model_generators/test_requirements_model_generator/test_microservice_requirements_model_generator/requirements.yml +++ b/tests/resources/unit/reqstool/model_generators/test_requirements_model_generator/test_microservice_requirements_model_generator/requirements.yml @@ -8,14 +8,14 @@ metadata: imports: git: - - token: "GITLAB_TOKEN" + - token: "dummy-gitlab-token" url: https://gitlab.ms-example.com ref: main path: /some/ms-path local: - path: /some/local-ms-path maven: - - token: "MAVEN_TOKEN" + - token: "dummy-maven-token" url: https://repo.maven.org group_id: com.example.one artifact_id: test-one diff --git a/tests/resources/unit/reqstool/model_generators/test_requirements_model_generator/test_npm_requirements_model_generator/requirements.yml b/tests/resources/unit/reqstool/model_generators/test_requirements_model_generator/test_npm_requirements_model_generator/requirements.yml index cbc5a066..0b7e6bf5 100644 --- a/tests/resources/unit/reqstool/model_generators/test_requirements_model_generator/test_npm_requirements_model_generator/requirements.yml +++ b/tests/resources/unit/reqstool/model_generators/test_requirements_model_generator/test_npm_requirements_model_generator/requirements.yml @@ -7,7 +7,7 @@ metadata: imports: npm: - - token: "NPM_TOKEN" + - token: "dummy-npm-token" url: https://my.registry.example.com package: "@my-org/my-service-reqstool" version: 1.2.3 diff --git a/tests/resources/unit/reqstool/model_generators/test_requirements_model_generator/test_system_requirements_model_generator/requirements.yml b/tests/resources/unit/reqstool/model_generators/test_requirements_model_generator/test_system_requirements_model_generator/requirements.yml index da1e53fc..16a3a7cb 100644 --- a/tests/resources/unit/reqstool/model_generators/test_requirements_model_generator/test_system_requirements_model_generator/requirements.yml +++ b/tests/resources/unit/reqstool/model_generators/test_requirements_model_generator/test_system_requirements_model_generator/requirements.yml @@ -7,7 +7,7 @@ metadata: imports: git: - - token: "GITLAB_TOKEN" + - token: "dummy-gitlab-token" url: https://gitlab.sys-example.com ref: feature/sys path: /some/path @@ -15,7 +15,7 @@ imports: - path: /some/local-sys-path maven: - - token: "MAVEN_TOKEN" + - token: "dummy-maven-token" url: https://repo.maven.org group_id: com.example.one artifact_id: test-one @@ -27,7 +27,7 @@ imports: version: 0.0.2 pypi: - - token: "PYPI_TOKEN" + - token: "dummy-pypi-token" url: https://example.com/simple/ package: package1 version: 1.0.0 @@ -51,7 +51,7 @@ filters: implementations: # only used with type = system git: - - token: GITLAB_TOKEN + - token: "dummy-gitlab-token" url: https://gitlab.impl-example.com ref: feature/impl path: README.md @@ -59,14 +59,14 @@ implementations: # only used with type = system local: # for future use - path: /some/local-impl-path maven: - - token: "MAVEN_TOKEN" + - token: "dummy-maven-token" url: https://repo.maven.org group_id: com.example.one artifact_id: test-one version: 0.0.1 classifier: classifier1 - - token: "MAVEN_TOKEN" + - token: "dummy-maven-token" url: https://repo2.maven.org group_id: com.example.two artifact_id: test-two diff --git a/tests/unit/reqstool/common/test_locations.py b/tests/unit/reqstool/common/test_locations.py index e9f471e1..5b0349b3 100644 --- a/tests/unit/reqstool/common/test_locations.py +++ b/tests/unit/reqstool/common/test_locations.py @@ -35,7 +35,7 @@ def maven_location_data(): def test_git_location_data(git_location_data): - assert git_location_data.token == "GITLAB_TOKEN" + assert git_location_data.token.get_secret_value() == "GITLAB_TOKEN" assert git_location_data.ref == "main" assert git_location_data.url == "https://gitlab.example.com" assert git_location_data.path == "some/path" @@ -46,7 +46,7 @@ def test_local_location_data(local_location_data): def test_maven_location_data(maven_location_data): - assert maven_location_data.token == "MAVEN_TOKEN" + assert maven_location_data.token.get_secret_value() == "MAVEN_TOKEN" assert maven_location_data.url == "https://repo1.maven.org/maven2" assert maven_location_data.group_id == "com.example" assert maven_location_data.artifact_id == "artifactexample" diff --git a/tests/unit/reqstool/locations/test_git_location.py b/tests/unit/reqstool/locations/test_git_location.py index 78490826..27f697a4 100644 --- a/tests/unit/reqstool/locations/test_git_location.py +++ b/tests/unit/reqstool/locations/test_git_location.py @@ -19,7 +19,7 @@ def test_git_location(): path=PATH, ) - assert git_location.token == "GITLAB_TOKEN" + assert git_location.token.get_secret_value() == "GITLAB_TOKEN" assert git_location.url == "https://git.example.com/example/repo.git" assert git_location.ref == "main" assert git_location.path == PATH @@ -31,7 +31,7 @@ def test_git_location(): path=PATH, ) - assert git_location.token == "CI_TOKEN" + assert git_location.token.get_secret_value() == "CI_TOKEN" assert git_location.url == "https://git.example.com/repo.git" assert git_location.ref == "v1.2.0" assert git_location.path == PATH @@ -161,3 +161,30 @@ def test_git_location_make_available_git_error_treated_as_not_found(tmp_path): with patch("reqstool.locations.git_location.clone_repository", return_value=mock_repo): with pytest.raises(GitRefNotFoundError): git_location._make_available_on_localdisk(str(tmp_path)) + + +def test_git_location_make_available_with_token(tmp_path): + git_location = GitLocation(url="https://git.example.com/repo.git", ref="main", path="", token="secret-token") + mock_repo = _mock_repo(tmp_path) + + with patch("reqstool.locations.git_location.clone_repository", return_value=mock_repo) as mock_clone: + git_location._make_available_on_localdisk(str(tmp_path)) + + callbacks = mock_clone.call_args[1]["callbacks"] + assert callbacks is not None + assert callbacks.api_token == "secret-token" + + +def test_git_location_make_available_no_token_passes_no_callbacks(tmp_path): + git_location = GitLocation(url="https://git.example.com/repo.git", ref="main", path="") + mock_repo = _mock_repo(tmp_path) + + with patch("reqstool.locations.git_location.clone_repository", return_value=mock_repo) as mock_clone: + git_location._make_available_on_localdisk(str(tmp_path)) + + assert mock_clone.call_args[1]["callbacks"] is None + + +def test_git_location_token_not_in_repr(): + loc = GitLocation(url="https://git.example.com/repo.git", ref="main", token="super-secret") + assert "super-secret" not in repr(loc) diff --git a/tests/unit/reqstool/locations/test_maven_location.py b/tests/unit/reqstool/locations/test_maven_location.py index caad1259..bb2b65dd 100644 --- a/tests/unit/reqstool/locations/test_maven_location.py +++ b/tests/unit/reqstool/locations/test_maven_location.py @@ -16,8 +16,27 @@ def test_maven_location_make_available_no_token(tmp_path): mock_downloader.download.return_value = True extracted = str(tmp_path / "extracted") with ( - patch("reqstool.locations.maven_location.Downloader", return_value=mock_downloader), + patch("reqstool.locations.maven_location.Downloader", return_value=mock_downloader) as mock_dl, patch("reqstool.locations.maven_location.Utils.extract_zip", return_value=extracted), ): result = loc._make_available_on_localdisk(str(tmp_path)) assert result == extracted + mock_dl.assert_called_once_with(base=loc.url, token=None) + + +def test_maven_location_make_available_with_token(tmp_path): + loc = MavenLocation(group_id="com.example", artifact_id="my-lib", version="1.0.0", token="my-secret") + mock_downloader = MagicMock() + mock_downloader.download.return_value = True + extracted = str(tmp_path / "extracted") + with ( + patch("reqstool.locations.maven_location.Downloader", return_value=mock_downloader) as mock_dl, + patch("reqstool.locations.maven_location.Utils.extract_zip", return_value=extracted), + ): + loc._make_available_on_localdisk(str(tmp_path)) + mock_dl.assert_called_once_with(base=loc.url, token="my-secret") + + +def test_maven_location_token_not_in_repr(): + loc = MavenLocation(group_id="com.example", artifact_id="my-lib", version="1.0.0", token="super-secret") + assert "super-secret" not in repr(loc) diff --git a/tests/unit/reqstool/locations/test_npm_location.py b/tests/unit/reqstool/locations/test_npm_location.py index 437b6310..e15b4e47 100644 --- a/tests/unit/reqstool/locations/test_npm_location.py +++ b/tests/unit/reqstool/locations/test_npm_location.py @@ -204,6 +204,25 @@ def test_npm_location_get_tarball_uses_timeout(tmp_path): assert mock_get.call_args[1].get("timeout") == 30 +def test_npm_location_make_available_empty_string_token_treated_as_no_auth(tmp_path): + loc = NpmLocation(package="my-pkg-reqstool", version="1.0.0", token="") + mock_response = MagicMock() + mock_response.json.return_value = {"dist": {"tarball": "https://registry.npmjs.org/tarball.tgz"}} + extracted = str(tmp_path / "package") + with ( + patch("reqstool.locations.npm_location.requests.get", return_value=mock_response) as mock_get, + patch("reqstool.locations.npm_location.Utils.download_file", return_value=tmp_path / "tarball.tgz"), + patch("reqstool.locations.npm_location.Utils.extract_targz", return_value=extracted), + ): + loc._make_available_on_localdisk(str(tmp_path)) + assert "Authorization" not in mock_get.call_args[1]["headers"] + + +def test_npm_location_token_not_in_repr(): + loc = NpmLocation(package="my-pkg-reqstool", version="1.0.0", token="super-secret") + assert "super-secret" not in repr(loc) + + def test_npm_location_make_available_no_token_sends_no_auth(tmp_path): loc = NpmLocation(package="my-pkg-reqstool", version="1.0.0") diff --git a/tests/unit/reqstool/locations/test_pypi_location.py b/tests/unit/reqstool/locations/test_pypi_location.py index 63445749..d995861c 100644 --- a/tests/unit/reqstool/locations/test_pypi_location.py +++ b/tests/unit/reqstool/locations/test_pypi_location.py @@ -14,9 +14,39 @@ def test_pypi_location_make_available_no_token(tmp_path): loc = PypiLocation(package="my-package", version="1.0.0") extracted = str(tmp_path / "extracted") with ( - patch.object(PypiLocation, "get_package_url", return_value="https://example.com/pkg.tar.gz"), + patch.object(PypiLocation, "get_package_url", return_value="https://example.com/pkg.tar.gz") as mock_url, patch("reqstool.locations.pypi_location.Utils.download_file", return_value=str(tmp_path / "pkg.tar.gz")), patch("reqstool.locations.pypi_location.Utils.extract_targz", return_value=extracted), ): result = loc._make_available_on_localdisk(str(tmp_path)) assert result == extracted + assert mock_url.call_args[0][3] is None + + +def test_pypi_location_make_available_with_token(tmp_path): + loc = PypiLocation(package="my-package", version="1.0.0", token="bearer-secret") + extracted = str(tmp_path / "extracted") + with ( + patch.object(PypiLocation, "get_package_url", return_value="https://example.com/pkg.tar.gz") as mock_url, + patch("reqstool.locations.pypi_location.Utils.download_file", return_value=str(tmp_path / "pkg.tar.gz")), + patch("reqstool.locations.pypi_location.Utils.extract_targz", return_value=extracted), + ): + loc._make_available_on_localdisk(str(tmp_path)) + assert mock_url.call_args[0][3] == "bearer-secret" + + +def test_pypi_location_make_available_empty_string_token_treated_as_no_auth(tmp_path): + loc = PypiLocation(package="my-package", version="1.0.0", token="") + extracted = str(tmp_path / "extracted") + with ( + patch.object(PypiLocation, "get_package_url", return_value="https://example.com/pkg.tar.gz") as mock_url, + patch("reqstool.locations.pypi_location.Utils.download_file", return_value=str(tmp_path / "pkg.tar.gz")), + patch("reqstool.locations.pypi_location.Utils.extract_targz", return_value=extracted), + ): + loc._make_available_on_localdisk(str(tmp_path)) + assert mock_url.call_args[0][3] is None + + +def test_pypi_location_token_not_in_repr(): + loc = PypiLocation(package="my-package", version="1.0.0", token="super-secret") + assert "super-secret" not in repr(loc) diff --git a/tests/unit/reqstool/model_generators/test_requirements_model_generator.py b/tests/unit/reqstool/model_generators/test_requirements_model_generator.py index 22e7e74e..30aa7f34 100644 --- a/tests/unit/reqstool/model_generators/test_requirements_model_generator.py +++ b/tests/unit/reqstool/model_generators/test_requirements_model_generator.py @@ -45,13 +45,13 @@ def test_system_requirements_model_generator(resource_funcname_rootdir_w_path): assert model.imports[0].current_unresolved.path == "/some/local-sys-path" # git - assert model.imports[1].current_unresolved.token == "GITLAB_TOKEN" + assert model.imports[1].current_unresolved.token.get_secret_value() == "dummy-gitlab-token" assert model.imports[1].current_unresolved.url == "https://gitlab.sys-example.com" assert model.imports[1].current_unresolved.ref == "feature/sys" assert model.imports[1].current_unresolved.path == "/some/path" # maven #1 - assert model.imports[2].current_unresolved.token == "MAVEN_TOKEN" + assert model.imports[2].current_unresolved.token.get_secret_value() == "dummy-maven-token" assert model.imports[2].current_unresolved.url == "https://repo.maven.org" assert model.imports[2].current_unresolved.group_id == "com.example.one" assert model.imports[2].current_unresolved.artifact_id == "test-one" @@ -67,7 +67,7 @@ def test_system_requirements_model_generator(resource_funcname_rootdir_w_path): assert model.imports[3].current_unresolved.classifier == "reqstool" # pypi #1 - assert model.imports[4].current_unresolved.token == "PYPI_TOKEN" + assert model.imports[4].current_unresolved.token.get_secret_value() == "dummy-pypi-token" assert model.imports[4].current_unresolved.url == "https://example.com/simple/" assert model.imports[4].current_unresolved.package == "package1" assert model.imports[4].current_unresolved.version == "1.0.0" @@ -85,7 +85,7 @@ def test_system_requirements_model_generator(resource_funcname_rootdir_w_path): assert model.implementations[0].current_unresolved.path == "/some/local-impl-path" # git - assert model.implementations[1].current_unresolved.token == "GITLAB_TOKEN" + assert model.implementations[1].current_unresolved.token.get_secret_value() == "dummy-gitlab-token" assert model.implementations[1].current_unresolved.url == "https://gitlab.impl-example.com" assert model.implementations[1].current_unresolved.ref == "feature/impl" assert model.implementations[1].current_unresolved.path == "README.md" @@ -147,7 +147,7 @@ def test_microservice_requirements_model_generator(resource_funcname_rootdir_w_p assert model.imports[0].current_unresolved.path == "/some/local-ms-path" # git - assert model.imports[1].current_unresolved.token == "GITLAB_TOKEN" + assert model.imports[1].current_unresolved.token.get_secret_value() == "dummy-gitlab-token" assert model.imports[1].current_unresolved.url == "https://gitlab.ms-example.com" assert model.imports[1].current_unresolved.ref == "main" assert model.imports[1].current_unresolved.path == "/some/ms-path" @@ -278,7 +278,7 @@ def test_npm_requirements_model_generator(resource_funcname_rootdir_w_path): # npm import #1 — with token and custom registry npm_import_1 = model.imports[0].current_unresolved assert isinstance(npm_import_1, NpmLocation) - assert npm_import_1.token == "NPM_TOKEN" + assert npm_import_1.token.get_secret_value() == "dummy-npm-token" assert str(npm_import_1.url) == "https://my.registry.example.com" assert npm_import_1.package == "@my-org/my-service-reqstool" assert npm_import_1.version == "1.2.3" diff --git a/tests/unit/reqstool/models/test_implementations.py b/tests/unit/reqstool/models/test_implementations.py index a0dbd6b6..4c39d678 100644 --- a/tests/unit/reqstool/models/test_implementations.py +++ b/tests/unit/reqstool/models/test_implementations.py @@ -48,7 +48,7 @@ def implementations_data(maven_impl_data): def test_git_impl_data(git_impl_data): assert git_impl_data.parent is None - assert git_impl_data.current.token == "GITLAB_TOKEN" + assert git_impl_data.current.token.get_secret_value() == "GITLAB_TOKEN" assert git_impl_data.current.ref == "main" assert git_impl_data.current.url == "https://github.com/reqstool/reqstool-client" assert git_impl_data.current.path == "/examples/README.adoc" @@ -61,7 +61,7 @@ def test_local_impl_data(local_impl_data): def test_maven_impl_data(maven_impl_data): assert maven_impl_data.parent is None - assert maven_impl_data.current.token == "MAVEN_TOKEN" + assert maven_impl_data.current.token.get_secret_value() == "MAVEN_TOKEN" assert maven_impl_data.current.url == "https://repo1.maven.org/maven2" assert maven_impl_data.current.group_id == "com.example" assert maven_impl_data.current.artifact_id == "artifactexample" diff --git a/tests/unit/reqstool/models/test_imports.py b/tests/unit/reqstool/models/test_imports.py index d8193a0a..5ef65b83 100644 --- a/tests/unit/reqstool/models/test_imports.py +++ b/tests/unit/reqstool/models/test_imports.py @@ -43,7 +43,7 @@ def maven_import_data(): def test_git_import_data(git_import_data): assert git_import_data.parent is None - assert git_import_data.current.token == "GITLAB_TOKEN" + assert git_import_data.current.token.get_secret_value() == "GITLAB_TOKEN" assert git_import_data.current.ref == "main" assert git_import_data.current.url == "https://gitlab.example.com" assert git_import_data.current.path == "git/some/path" @@ -56,7 +56,7 @@ def test_local_system_data(local_import_data): def test_maven_system_data(maven_import_data): assert maven_import_data.parent is None - assert maven_import_data.current.token == "MAVEN_TOKEN" + assert maven_import_data.current.token.get_secret_value() == "MAVEN_TOKEN" assert maven_import_data.current.url == "https://repo1.maven.org/maven2" assert maven_import_data.current.group_id == "com.example" assert maven_import_data.current.artifact_id == "artifactexample" diff --git a/tests/unit/reqstool/test_command.py b/tests/unit/reqstool/test_command.py index 1986225c..e1d228a9 100644 --- a/tests/unit/reqstool/test_command.py +++ b/tests/unit/reqstool/test_command.py @@ -13,6 +13,9 @@ EXIT_CODE_MISSING_REQUIREMENTS_FILE, ) from reqstool.common.exceptions import ArtifactDownloadError, MissingRequirementsFileError +from reqstool.locations.git_location import GitLocation +from reqstool.locations.maven_location import MavenLocation +from reqstool.locations.pypi_location import PypiLocation from reqstool.locations.local_npm_location import LocalNpmLocation from reqstool.locations.npm_location import NpmLocation @@ -142,6 +145,58 @@ def test_git_source_parser_requires_url_path_and_ref(): assert args.ref == "v1.0.0" +def test_git_source_parser_accepts_token(): + args = _make_command_and_parse( + ["reqstool", "report", "git", "-u", "https://example.com/repo", "-p", "docs", "-r", "main", "-t", "secret"] + ) + assert args.token == "secret" + + +def test_maven_source_parser_accepts_token(): + args = _make_command_and_parse( + ["reqstool", "report", "maven", "--group_id", "com.ex", "--artifact_id", "lib", "--version", "1.0.0", "-t", "s"] + ) + assert args.token == "s" + + +def test_pypi_source_parser_accepts_token(): + args = _make_command_and_parse( + ["reqstool", "report", "pypi", "--package", "mypkg", "--version", "1.0.0", "-t", "secret"] + ) + assert args.token == "secret" + + +def test_get_initial_source_git_with_token(): + args = argparse.Namespace(source="git", url="https://example.com/repo.git", path="docs", ref="main", token="sec") + loc = Command()._get_initial_source(args) + assert isinstance(loc, GitLocation) + assert loc.token.get_secret_value() == "sec" + + +def test_get_initial_source_maven_with_token(): + args = argparse.Namespace( + source="maven", + url=None, + group_id="com.example", + artifact_id="lib", + version="1.0.0", + classifier="reqstool", + token="sec", + ) + loc = Command()._get_initial_source(args) + assert isinstance(loc, MavenLocation) + assert loc.token.get_secret_value() == "sec" + + +def test_get_initial_source_pypi_with_token(): + args = argparse.Namespace( + source="pypi", url="https://pypi.org/simple", package="mypkg", version="1.0.0", token="sec" + ) + loc = Command()._get_initial_source(args) + assert isinstance(loc, PypiLocation) + assert loc.token.get_secret_value() == "sec" + + def test_git_source_parser_missing_ref_errors(): with pytest.raises(SystemExit): _make_command_and_parse(["reqstool", "report", "git", "-u", "https://example.com/repo", "-p", "docs/reqstool"]) @@ -230,7 +285,7 @@ def test_get_initial_source_npm_with_custom_url(): loc = Command()._get_initial_source(args) assert isinstance(loc, NpmLocation) assert loc.url == "https://my.registry.example.com" - assert loc.token == "NPM_TOKEN" + assert loc.token.get_secret_value() == "NPM_TOKEN" def test_get_initial_source_local_npm_returns_local_npm_location(): From 1a7bc4a6a57c9319529eb92097e840e9b2592b1d Mon Sep 17 00:00:00 2001 From: Jimisola Laursen Date: Tue, 2 Jun 2026 08:21:25 +0200 Subject: [PATCH 4/5] fix(testdata): handle Gradle parameterized tests and fix worst-status-wins Gradle JUnit 5 parameterized tests produce display-name-only test case names ([N] args or N text) without any method identifier. The parser previously hit the else branch for these, logging ERROR and writing all instances to classname.invalid_method_name, polluting the FQN index. Add DISPLAY_NAME_INDEX_REGEX to detect these formats and skip them with WARNING instead. Also fix last-write-wins to worst-status-wins so that a FAILED instance of any Maven-style parameterized test (methodName(T)[N]) is not silently overwritten by a later PASSED instance. Maven Surefire always embeds the method name (methodName(Type)[N]) and is unaffected. This limitation is Gradle-specific. Closes #403 Signed-off-by: Jimisola Laursen --- .../testdata_model_generator.py | 28 +++++++++--- .../TEST-com.example.StatusServiceTest.xml | 17 ++++++++ .../test_testdata_model_generator.py | 43 +++++++++++++++++++ 3 files changed, 83 insertions(+), 5 deletions(-) create mode 100644 tests/resources/unit/reqstool/model_generators/test_testdata_model_generator/test_parameterized_testdata_model_generator/TEST-com.example.StatusServiceTest.xml diff --git a/src/reqstool/model_generators/testdata_model_generator.py b/src/reqstool/model_generators/testdata_model_generator.py index d09139c9..86271ca1 100644 --- a/src/reqstool/model_generators/testdata_model_generator.py +++ b/src/reqstool/model_generators/testdata_model_generator.py @@ -14,9 +14,15 @@ from reqstool.models.test_data import TEST_RUN_STATUS, TestData, TestsData +def _status_priority(status: TEST_RUN_STATUS) -> int: + return {TEST_RUN_STATUS.PASSED: 0, TEST_RUN_STATUS.SKIPPED: 1, TEST_RUN_STATUS.FAILED: 2}.get(status, 0) + + class TestDataModelGenerator: UNIT_METHOD_IDENTIFIER_REGEX = r"^([a-zA-Z_$][a-zA-Z0-9_$]*).*$" KARATE_METHOD_IDENTIFIER_REGEX = r"\[\d+(?:\.\d+)?:\d+\]\s*(.+)" + # Gradle parameterized: default "[N] args" or custom "{index} text" (N text) — method name absent + DISPLAY_NAME_INDEX_REGEX = r"^\[?\d" def __init__(self, test_result_files: List[Path], urn: str): self.test_result_files = test_result_files @@ -29,8 +35,8 @@ def __generate(self, test_result_files: List[Path], urn: str) -> TestsData: return TestsData(tests=tests) @Requirements("REQ_014", "REQ_015") - def __parse_test_data(self, test_result_files: List[Path], urn: str) -> Dict[str, TestData]: - r_testdata: Dict[str, TestData] = {} + def __parse_test_data(self, test_result_files: List[Path], urn: str) -> Dict[UrnId, TestData]: + r_testdata: Dict[UrnId, TestData] = {} for test_result_file in test_result_files: @@ -50,9 +56,19 @@ def __parse_test_data(self, test_result_files: List[Path], urn: str) -> Dict[str methodname = match_unit.group(1) elif match_karate: methodname = match_karate.group(1) + elif re.match(self.DISPLAY_NAME_INDEX_REGEX, testcase.attrib["name"]): + logging.warning( + f"Skipping parameterized test case with display-name-only format " + f"(method name not recoverable): {testcase.attrib['name']!r} " + f"in {test_result_file}" + ) + continue else: - logging.error(f"{testcase.attrib['name']} is not a valid method name\n") - methodname = "invalid_method_name" + logging.warning( + f"Skipping test case with unrecognized name format: " + f"{testcase.attrib['name']!r} in {test_result_file}" + ) + continue test_run_status: TEST_RUN_STATUS @@ -67,6 +83,8 @@ def __parse_test_data(self, test_result_files: List[Path], urn: str) -> Dict[str test_data = TestData(fully_qualified_name=fqn, status=test_run_status) urn_id = UrnId(urn=urn, id=fqn) - r_testdata[urn_id] = test_data + existing = r_testdata.get(urn_id) + if existing is None or _status_priority(test_run_status) > _status_priority(existing.status): + r_testdata[urn_id] = test_data return r_testdata diff --git a/tests/resources/unit/reqstool/model_generators/test_testdata_model_generator/test_parameterized_testdata_model_generator/TEST-com.example.StatusServiceTest.xml b/tests/resources/unit/reqstool/model_generators/test_testdata_model_generator/test_parameterized_testdata_model_generator/TEST-com.example.StatusServiceTest.xml new file mode 100644 index 00000000..c51c4ff5 --- /dev/null +++ b/tests/resources/unit/reqstool/model_generators/test_testdata_model_generator/test_parameterized_testdata_model_generator/TEST-com.example.StatusServiceTest.xml @@ -0,0 +1,17 @@ + + + + + + + + + + + + + + + + diff --git a/tests/unit/reqstool/model_generators/test_testdata_model_generator.py b/tests/unit/reqstool/model_generators/test_testdata_model_generator.py index d3699860..4d0573f1 100644 --- a/tests/unit/reqstool/model_generators/test_testdata_model_generator.py +++ b/tests/unit/reqstool/model_generators/test_testdata_model_generator.py @@ -1,11 +1,14 @@ # Copyright © LFV +import logging import re import pytest from reqstool_python_decorators.decorators.decorators import SVCs +from reqstool.common.models.urn_id import UrnId from reqstool.model_generators.testdata_model_generator import TestDataModelGenerator +from reqstool.models.test_data import TEST_RUN_STATUS karate_method_names = [ "[1.4:55] Create a subscripiton with filter and receive messages", @@ -35,6 +38,46 @@ def test_unit_method_identifier_regex(method_name): assert unit_match.group(1) == "testFlightIdAircraftId" +display_name_index_names = [ + "[1] ACTIVE", + "[2] conflictMessage=Entry 10 is AWAITING_MANUAL_EDIT, expected PENDING", + "1 status=ACTIVE", + "2 someDisplayName", +] + + +@pytest.mark.parametrize("method_name", display_name_index_names) +def test_display_name_index_regex(method_name): + assert re.match(TestDataModelGenerator.UNIT_METHOD_IDENTIFIER_REGEX, method_name) is None + assert re.match(TestDataModelGenerator.KARATE_METHOD_IDENTIFIER_REGEX, method_name) is None + assert re.match(TestDataModelGenerator.DISPLAY_NAME_INDEX_REGEX, method_name) is not None + + +def test_parameterized_testdata_model_generator(resource_funcname_rootdir_w_path, caplog): + with caplog.at_level(logging.WARNING): + tdmg = TestDataModelGenerator( + test_result_files=[ + resource_funcname_rootdir_w_path("TEST-com.example.StatusServiceTest.xml") + ], + urn="test", + ) + + tests = tdmg.model.tests + + # Only the Maven-style method is in the index; Gradle display-name entries are skipped + assert len(tests) == 1 + + # checkStatusMaven: [1] PASSED, [2] PASSED, [3] FAILED → worst-status-wins → FAILED + fqn = "com.example.StatusServiceTest.checkStatusMaven" + urn_id = UrnId(urn="test", id=fqn) + assert urn_id in tests + assert tests[urn_id].status == TEST_RUN_STATUS.FAILED + + # Gradle display-name entries produced warnings, not errors + assert any("display-name-only" in r.message for r in caplog.records) + assert all(r.levelname != "ERROR" for r in caplog.records) + + def test_testdata_model_generator(local_testdata_resources_rootdir_w_path): # TODO: # * Test the different variants: passed, skipped, failure etc From c5a42117fab8b075a2edd3940188fbe879d8abbb Mon Sep 17 00:00:00 2001 From: Jimisola Laursen Date: Tue, 2 Jun 2026 10:11:58 +0200 Subject: [PATCH 5/5] style: apply black formatting to test_testdata_model_generator.py Signed-off-by: Jimisola Laursen --- .../model_generators/test_testdata_model_generator.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tests/unit/reqstool/model_generators/test_testdata_model_generator.py b/tests/unit/reqstool/model_generators/test_testdata_model_generator.py index 4d0573f1..16176b65 100644 --- a/tests/unit/reqstool/model_generators/test_testdata_model_generator.py +++ b/tests/unit/reqstool/model_generators/test_testdata_model_generator.py @@ -56,9 +56,7 @@ def test_display_name_index_regex(method_name): def test_parameterized_testdata_model_generator(resource_funcname_rootdir_w_path, caplog): with caplog.at_level(logging.WARNING): tdmg = TestDataModelGenerator( - test_result_files=[ - resource_funcname_rootdir_w_path("TEST-com.example.StatusServiceTest.xml") - ], + test_result_files=[resource_funcname_rootdir_w_path("TEST-com.example.StatusServiceTest.xml")], urn="test", )