Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 3 additions & 5 deletions docs/hooks.rst
Original file line number Diff line number Diff line change
Expand Up @@ -230,9 +230,7 @@ Source hooks
.. autofromagerhook:: default_build_sdist

The ``build_sdist()`` function is responsible for creating a new source
distribution from the prepared source tree and placing it in
``ctx.sdists_build``. The dist name and version of the sdist file must
match the ``Requirement`` and ``Version``.
distribution from the prepared source tree and placing it in ``ctx.sdists_build``.

The arguments are the ``WorkContext``, the ``Requirement`` being evaluated, and the
`Path` to the root of the source tree.
Expand All @@ -249,8 +247,8 @@ Wheel hooks

The ``build_wheel()`` function is responsible for creating a wheel from
the prepared source tree and placing it in ``ctx.wheels_build``. The
default implementation uses :pep:`517` pyproject hook. The dist name
and version of the wheel must match the ``Requirement`` and ``Version``.
default implementation invokes ``pip wheel`` in a temporary directory
and passes the path to the source tree as argument.

The arguments are the ``WorkContext``, the ``Path`` to a virtualenv
prepared with the build dependencies, a ``dict`` with extra environment
Expand Down
39 changes: 3 additions & 36 deletions src/fromager/commands/lint_requirements.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,9 +5,7 @@
import click
from packaging.requirements import InvalidRequirement, Requirement

from fromager import bootstrapper, context, progress, requirements_file
from fromager.log import requirement_ctxvar
from fromager.requirements_file import RequirementType
from fromager import requirements_file

logger = logging.getLogger(__name__)

Expand All @@ -19,17 +17,13 @@
required=True,
type=click.Path(exists=False, path_type=pathlib.Path),
)
@click.pass_obj
def lint_requirements(
wkctx: context.WorkContext, input_files_path: list[pathlib.Path]
) -> None:
def lint_requirements(input_files_path: list[pathlib.Path]) -> None:
"""
Command to lint the constraints.txt and requirements.txt files
This command takes a single wildcard path string for constraints.txt and requirements.txt.
It checks the formatting of these files and reports issues if found. Files with names that
end with constraints.txt (e.g. constraints.txt, global-constraints.txt, etc.) are not allowed
to contain extra dependencies. Additionally, it resolves valid input requirements to ensure
we can find a matching version of each package.
to contain extra dependencies.
"""

if len(input_files_path) == 0:
Expand All @@ -38,15 +32,6 @@ def lint_requirements(

flag = True

# Create bootstrapper for requirement resolution
bt = bootstrapper.Bootstrapper(
ctx=wkctx,
progressbar=progress.Progressbar(None),
prev_graph=None,
cache_wheel_server_url=None,
sdist_only=True,
)

for path in input_files_path:
parsed_lines = requirements_file.parse_requirements_file(path)
unique_entries: dict[str, Requirement] = {}
Expand All @@ -62,24 +47,6 @@ def lint_requirements(
raise InvalidRequirement(
"Constraints files cannot contain extra dependencies"
)

# Resolve the requirement to ensure it can be found
# Skip resolution for constraints files as they should only specify versions
if not path.name.endswith("constraints.txt"):
token = requirement_ctxvar.set(requirement)
try:
_, version = bt.resolve_version(
req=requirement,
req_type=RequirementType.TOP_LEVEL,
)
logger.info(f"{requirement} resolves to {version}")
except Exception as resolve_err:
logger.error(
f"{path}: {line}: Failed to resolve requirement: {resolve_err}"
)
flag = False
finally:
requirement_ctxvar.reset(token)
except InvalidRequirement as err:
logger.error(f"{path}: {line}: {err}")
flag = False
Expand Down
6 changes: 3 additions & 3 deletions src/fromager/context.py
Original file line number Diff line number Diff line change
Expand Up @@ -63,15 +63,15 @@ def __init__(
self.constraints.load_constraints_file(constraints_file)
else:
self.input_constraints_uri = None
self.sdists_repo = pathlib.Path(sdists_repo).resolve()
self.sdists_repo = pathlib.Path(sdists_repo).absolute()
self.sdists_downloads = self.sdists_repo / "downloads"
self.sdists_builds = self.sdists_repo / "builds"
self.wheels_repo = pathlib.Path(wheels_repo).resolve()
self.wheels_repo = pathlib.Path(wheels_repo).absolute()
self.wheels_build_base = self.wheels_repo / "build"
self.wheels_downloads = self.wheels_repo / "downloads"
self.wheels_prebuilt = self.wheels_repo / "prebuilt"
self.wheel_server_dir = self.wheels_repo / "simple"
self.work_dir = pathlib.Path(work_dir).resolve()
self.work_dir = pathlib.Path(work_dir).absolute()
self.graph_file = self.work_dir / "graph.json"
self.uv_cache = self.work_dir / "uv-cache"
self.wheel_server_url = wheel_server_url
Expand Down
88 changes: 11 additions & 77 deletions src/fromager/dependencies.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,6 @@
import tomlkit
from packaging.metadata import Metadata
from packaging.requirements import Requirement
from packaging.utils import NormalizedName, canonicalize_name
from packaging.version import Version

from . import (
Expand Down Expand Up @@ -303,25 +302,21 @@ def default_get_install_dependencies_of_sdist(

Uses PEP 517 prepare_metadata_for_build_wheel() API.
"""
metadata = pep517_metadata_of_sdist(
hook_caller = get_build_backend_hook_caller(
ctx=ctx,
req=req,
version=version,
sdist_root_dir=sdist_root_dir,
build_env=build_env,
extra_environ=extra_environ,
build_dir=build_dir,
config_settings=config_settings,
validate=True,
)
validate_dist_name_version(
req=req,
version=version,
what="sdist metadata",
# Metadata name is a non-normalized string
dist_name=canonicalize_name(metadata.name),
dist_version=metadata.version,
override_environ=extra_environ,
build_env=build_env,
)
with tempfile.TemporaryDirectory() as tmp_dir:
distinfo_name = hook_caller.prepare_metadata_for_build_wheel(
tmp_dir,
config_settings=config_settings,
)
metadata_file = pathlib.Path(tmp_dir) / distinfo_name / "METADATA"
# ignore minor metadata issues
metadata = parse_metadata(metadata_file, validate=False)
if not metadata.requires_dist:
return set()
return set(metadata.requires_dist)
Expand All @@ -337,67 +332,6 @@ def parse_metadata(metadata_file: pathlib.Path, *, validate: bool = True) -> Met
return Metadata.from_email(metadata_file.read_bytes(), validate=validate)


def pep517_metadata_of_sdist(
*,
ctx: context.WorkContext,
req: Requirement,
version: Version,
sdist_root_dir: pathlib.Path,
build_env: build_environment.BuildEnvironment,
extra_environ: dict[str, str],
build_dir: pathlib.Path,
config_settings: dict[str, str],
validate: bool = True,
) -> Metadata:
"""Get wheel metadata from a source distribution

Uses PEP 517 prepare_metadata_for_build_wheel() API.
"""
hook_caller = get_build_backend_hook_caller(
ctx=ctx,
req=req,
build_dir=build_dir,
override_environ=extra_environ,
build_env=build_env,
)
with tempfile.TemporaryDirectory() as tmp_dir:
distinfo_name = hook_caller.prepare_metadata_for_build_wheel(
tmp_dir,
config_settings=config_settings,
)
metadata_file = pathlib.Path(tmp_dir) / distinfo_name / "METADATA"
metadata = parse_metadata(metadata_file, validate=validate)

return metadata


def validate_dist_name_version(
req: Requirement,
version: Version,
what: str,
dist_name: NormalizedName,
dist_version: Version,
) -> None:
"""Validate that dist name and version matches expected values"""
req_name = canonicalize_name(req.name)
if dist_name != req_name:
if dist_name != canonicalize_name(dist_name):
# API misuse
raise RuntimeError("dist_name argument {dist_name!r} is not normalized")
else:
raise ValueError(
f"{what} {dist_name!r} does not match requirement {req_name!r}"
)
if dist_version.public != version.public:
raise ValueError(
f"{what} {dist_version.public!r} does not match public version {version.public!r}"
)
if dist_version.local != version.local:
logger.warning(
f"{what} {dist_version.local!r} has different local version than {version.local!r}"
)


def get_install_dependencies_of_wheel(
req: Requirement, wheel_filename: pathlib.Path, requirements_file_dir: pathlib.Path
) -> set[Requirement]:
Expand Down
38 changes: 4 additions & 34 deletions src/fromager/sources.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,9 +11,6 @@

import resolvelib
from packaging.requirements import Requirement
from packaging.utils import (
parse_sdist_filename,
)
from packaging.version import Version
from requests.exceptions import ChunkedEncodingError, ConnectionError
from urllib3.exceptions import IncompleteRead, ProtocolError
Expand Down Expand Up @@ -587,7 +584,7 @@ def build_sdist(
# NOT produce a valid sdist, so we can use the PEP-517 approach
# instead.
logger.info("using PEP-517 sdist build, ignoring any plugins")
sdist_file = pep517_build_sdist(
sdist_filename = pep517_build_sdist(
ctx=ctx,
extra_environ=extra_environ,
req=req,
Expand All @@ -596,7 +593,7 @@ def build_sdist(
build_env=build_env,
)
else:
sdist_file = overrides.find_and_invoke(
sdist_filename = overrides.find_and_invoke(
req.name,
"build_sdist",
default_build_sdist,
Expand All @@ -608,19 +605,8 @@ def build_sdist(
build_dir=build_dir,
build_env=build_env,
)
logger.info(f"built source distribution {sdist_file}")

# validate location and file name
if not sdist_file.is_file():
raise FileNotFoundError(sdist_file)
sdist_file = sdist_file.resolve()
if sdist_file.parent != ctx.sdists_builds:
raise ValueError(
f"{sdist_file!r} is not in sdists build directory {ctx.sdists_builds!r}"
)
validate_sdist_filename(req=req, version=version, sdist_file=sdist_file)

return sdist_file
logger.info(f"built source distribution {sdist_filename}")
return sdist_filename


def default_build_sdist(
Expand Down Expand Up @@ -730,19 +716,3 @@ def ensure_pkg_info(
)
had_pkg_info = False
return had_pkg_info


def validate_sdist_filename(
req: Requirement,
version: Version,
sdist_file: pathlib.Path,
) -> None:
"""Check that sdist matches requirement name and version"""
sdist_name, sdist_version = parse_sdist_filename(sdist_file.name)
dependencies.validate_dist_name_version(
req=req,
version=version,
what=sdist_file.name,
dist_name=sdist_name,
dist_version=sdist_version,
)
Loading
Loading