From 44bc4b144b592bbc1621c7788b930c9dc69948d9 Mon Sep 17 00:00:00 2001 From: Christian Heimes Date: Thu, 25 Sep 2025 12:14:34 +0200 Subject: [PATCH] feat(lint-requirements): make resolving requirements optional In downstream we are having issues with new resolve version feature of the linter. Some packages need extra configuration to resolve versions from git repos. - disable resolving of requirements by default - add flag `--resolve-requirements` (default: False) to enable resolving - collect all failures and print a summary at the end - refactor code a bit Signed-off-by: Christian Heimes --- e2e/test_lint_requirements.sh | 8 +-- src/fromager/commands/lint_requirements.py | 75 ++++++++++++++-------- 2 files changed, 54 insertions(+), 29 deletions(-) diff --git a/e2e/test_lint_requirements.sh b/e2e/test_lint_requirements.sh index c5275c43..bb187529 100755 --- a/e2e/test_lint_requirements.sh +++ b/e2e/test_lint_requirements.sh @@ -10,7 +10,7 @@ pass=true # Test to demonstrate that command works as expected when input files are valid -if ! fromager lint-requirements "$SCRIPTDIR/validate_inputs/constraints.txt" "$SCRIPTDIR/validate_inputs/requirements.txt"; then +if ! fromager lint-requirements --resolve-requirements "$SCRIPTDIR/validate_inputs/constraints.txt" "$SCRIPTDIR/validate_inputs/requirements.txt"; then echo "the input files should have been recognized as correctly formatted" 1>&2 pass=false fi; @@ -24,21 +24,21 @@ fi; # Test to demonstrate that command reports error for invalid / bad requirements -if fromager lint-requirements "$SCRIPTDIR/validate_inputs/constraints.txt" "$SCRIPTDIR/validate_inputs/invalid-requirements.txt"; then +if fromager lint-requirements --resolve-requirements "$SCRIPTDIR/validate_inputs/constraints.txt" "$SCRIPTDIR/validate_inputs/invalid-requirements.txt"; then echo "invalid input files should have been recognized by the command" 1>&2 pass=false fi; # Test to demonstrate that command reports error for invalid / bad constraints -if fromager lint-requirements "$SCRIPTDIR/validate_inputs/invalid-constraints.txt" "$SCRIPTDIR/validate_inputs/requirements.txt"; then +if fromager lint-requirements --resolve-requirements "$SCRIPTDIR/validate_inputs/invalid-constraints.txt" "$SCRIPTDIR/validate_inputs/requirements.txt"; then echo "invalid input files should have been recognized by the command" 1>&2 pass=false fi; # Test to demonstrate that command reports error for duplicate entries in files -if fromager lint-requirements "$SCRIPTDIR/validate_inputs/constraints.txt" "$SCRIPTDIR/validate_inputs/duplicate-requirements.txt"; then +if fromager lint-requirements --resolve-requirements "$SCRIPTDIR/validate_inputs/constraints.txt" "$SCRIPTDIR/validate_inputs/duplicate-requirements.txt"; then echo "duplicate entries in files should have been recognized by the command" 1>&2 pass=false fi; diff --git a/src/fromager/commands/lint_requirements.py b/src/fromager/commands/lint_requirements.py index 3aa54ea5..aa0abf9a 100644 --- a/src/fromager/commands/lint_requirements.py +++ b/src/fromager/commands/lint_requirements.py @@ -19,9 +19,17 @@ required=True, type=click.Path(exists=False, path_type=pathlib.Path), ) +@click.option( + "--resolve-requirements/--no-resolve-requirements", + default=False, + help="Resolve requirement and fail if a package or version cannot be resolved", + show_default=True, +) @click.pass_obj def lint_requirements( - wkctx: context.WorkContext, input_files_path: list[pathlib.Path] + wkctx: context.WorkContext, + resolve_requirements: bool, + input_files_path: list[pathlib.Path], ) -> None: """ Command to lint the constraints.txt and requirements.txt files @@ -36,7 +44,7 @@ def lint_requirements( logger.error("no constraints.txt or requirements.txt found in given paths") sys.exit(1) - flag = True + failures: list[str] = [] # Create bootstrapper for requirement resolution bt = bootstrapper.Bootstrapper( @@ -48,6 +56,7 @@ def lint_requirements( ) for path in input_files_path: + is_constraints: bool = path.name.endswith("constraints.txt") parsed_lines = requirements_file.parse_requirements_file(path) unique_entries: dict[str, Requirement] = {} for line in parsed_lines: @@ -58,31 +67,47 @@ def lint_requirements( f"Duplicate entry, first found: {unique_entries[requirement.name]}" ) unique_entries[requirement.name] = requirement - if requirement.extras and path.name.endswith("constraints.txt"): - raise InvalidRequirement( - "Constraints files cannot contain extra dependencies" - ) - - # Resolve the requirement to ensure it can be found - # Skip resolution for constraints files as they should only specify versions - if not path.name.endswith("constraints.txt"): - token = requirement_ctxvar.set(requirement) - try: - _, version = bt.resolve_version( - req=requirement, - req_type=RequirementType.TOP_LEVEL, + if is_constraints: + if requirement.extras: + raise InvalidRequirement( + f"{requirement.name}: Constraints files cannot contain extra dependencies" ) - logger.info(f"{requirement} resolves to {version}") - except Exception as resolve_err: - logger.error( - f"{path}: {line}: Failed to resolve requirement: {resolve_err}" + if not requirement.specifier: + raise InvalidRequirement( + f"{requirement.name}: Constraints must have a version specifier" ) - flag = False - finally: - requirement_ctxvar.reset(token) except InvalidRequirement as err: - logger.error(f"{path}: {line}: {err}") - flag = False + msg = f"{path}: {line}: {err}" + logger.error(msg) + failures.append(msg) + + # Resolve the requirement to ensure it can be found + # Skip resolution for constraints files as they should only specify versions + if resolve_requirements and not is_constraints: + token = requirement_ctxvar.set(requirement) + try: + _, version = bt.resolve_version( + req=requirement, + req_type=RequirementType.TOP_LEVEL, + ) + logger.info(f"{requirement} resolves to {version}") + except Exception as err: + logger.error( + f"{path}: {line}: Failed to resolve requirement: {err}" + ) + failures.append(f"{path}: {line}: {err}") + finally: + requirement_ctxvar.reset(token) - if not flag: + if failures: + click.echo("Validation error:", err=True) + for failure in failures: + click.echo(f" - {failure}", err=True) + click.echo( + f"ERROR: {len(failures)} failure(s) while validating {len(input_files_path)} file(s).", + err=True, + ) sys.exit(1) + else: + click.echo(f"Resolve requirements: {resolve_requirements}") + click.echo(f"Successfully validated {len(input_files_path)} file(s).")