From 5cc0602a4da0403fb1e6b5f117a44082fc63e7f6 Mon Sep 17 00:00:00 2001 From: Richard Levasseur Date: Fri, 26 Jun 2026 05:20:25 +0000 Subject: [PATCH] feat(pypi): add pip.dep to declare abstract pypi dependencies Introduce the pip.dep tag class to allow modules to declare abstract PyPI dependencies. These dependencies are fed directly into the unified hub repository, ensuring their target structures exist and automatically routing any unimplemented declarations to analysis-time errors. --- .agents/rules/bzl.md | 8 ++++ docs/pypi/download.md | 35 ++++++++++++++ news/pip-dep-tag-class.added.md | 4 ++ python/private/pypi/extension.bzl | 46 +++++++++++++++++++ tests/integration/unified_pypi/BUILD.bazel | 13 ++++++ tests/integration/unified_pypi/MODULE.bazel | 4 ++ .../unified_pypi/bin_declared_only.py | 2 + tests/integration/unified_pypi_test.py | 21 +++++++++ tests/pypi/extension/extension_tests.bzl | 39 +++++++++++++++- 9 files changed, 171 insertions(+), 1 deletion(-) create mode 100644 .agents/rules/bzl.md create mode 100644 news/pip-dep-tag-class.added.md create mode 100644 tests/integration/unified_pypi/bin_declared_only.py diff --git a/.agents/rules/bzl.md b/.agents/rules/bzl.md new file mode 100644 index 0000000000..104c52c590 --- /dev/null +++ b/.agents/rules/bzl.md @@ -0,0 +1,8 @@ +--- +trigger: glob +description: Starlark / Bazel .bzl file coding style rules +globs: *.bzl +--- + +* Use triple-quoted strings for multi-line rule doc args. +* Don't use backslash line continuation in rule doc args. diff --git a/docs/pypi/download.md b/docs/pypi/download.md index 6705df1f3a..ebdb63bf5f 100644 --- a/docs/pypi/download.md +++ b/docs/pypi/download.md @@ -121,6 +121,41 @@ Shared library targets can simply depend on the unified hub (e.g., `@pypi//numpy`), and the dependency will automatically resolve to the correct wheel version from the active hub during the build. +### Declaring Abstract Dependencies (pip.dep) + +:::{versionadded} VERSION_NEXT_FEATURE +Declaring abstract PyPI dependencies via `pip.dep` tags. +::: + +Sometimes a shared library target or a ruleset needs to depend on a PyPI +package (e.g., `@pypi//numpy`), but does not want to force a specific package +version or a concrete `requirements.txt` lock file on its consumers. + +Instead of calling `pip.parse()`, the module can declare its dependency using +the `pip.dep` tag: + +```starlark +pip = use_extension("@rules_python//python/extensions:pip.bzl", "pip") + +# Declare an abstract dependency on 'numpy' and specify extra targets that +# are expected to be available in the package. +pip.dep( + name = "numpy", + extra_targets = ["extra-alias"], +) +``` + +This ensures that the target structure `@pypi//numpy` (and +`@pypi//numpy:extra-alias`) exists in the unified `@pypi` hub repository, so the +declaring module can compile and analyze successfully without needing any local +requirements file. + +The actual concrete implementation and version of the package must be provided +by a downstream module calling `pip.parse`. + +If a downstream module attempts to build a target that depends on an abstract +dependency, but has not provided a concrete implementation for it via any +`pip.parse` call, the build will fail at execution time. As with any repository rule or extension, if you would like to ensure that `pip_parse` is diff --git a/news/pip-dep-tag-class.added.md b/news/pip-dep-tag-class.added.md new file mode 100644 index 0000000000..5306186081 --- /dev/null +++ b/news/pip-dep-tag-class.added.md @@ -0,0 +1,4 @@ +(pypi) Added a `dep` tag class to the `pip` bzlmod extension. This allows +modules to declare abstract PyPI dependencies, ensuring target structures +exist in the unified hub, while allowing other modules to provide the +concrete implementation via `pip.parse`. diff --git a/python/private/pypi/extension.bzl b/python/private/pypi/extension.bzl index 62d2c8de16..a3e775e08b 100644 --- a/python/private/pypi/extension.bzl +++ b/python/private/pypi/extension.bzl @@ -424,6 +424,15 @@ You cannot use both the additive_build_content and additive_build_content_file a pip_attr = pip_attr, ) + # dict[str package, dict[str, None] extra_targets] + declared_deps = {} + for mod in module_ctx.modules: + for dep_attr in mod.tags.dep: + name = normalize_name(dep_attr.name) + targets = declared_deps.setdefault(name, {}) + for target in dep_attr.extra_targets: + targets[target] = None + # Keeps track of all the hub's whl repos across the different versions. # dict[hub, dict[whl, dict[version, str pip]]] # Where hub, whl, and pip are the repo names @@ -448,6 +457,7 @@ You cannot use both the additive_build_content and additive_build_content_file a return struct( config = config, + declared_deps = declared_deps, default_hub = config.default_hub or renamed_default_hub, exposed_packages = exposed_packages, extra_aliases = extra_aliases, @@ -492,6 +502,16 @@ def _create_unified_hub_repo(mods): if hub_name not in extra_aliases[qual_alias]: extra_aliases[qual_alias].append(hub_name) + for dep_name, extra_targets in mods.declared_deps.items(): + norm_pkg = normalize_name(dep_name) + if norm_pkg not in packages: + packages[norm_pkg] = [] + + for target_name in extra_targets: + qual_alias = "%s:%s" % (norm_pkg, target_name) + if qual_alias not in extra_aliases: + extra_aliases[qual_alias] = [] + unified_hub_repo( name = "pypi", default_hub = mods.default_hub or (hubs[0] if hubs else ""), @@ -1015,6 +1035,31 @@ Apply any overrides (e.g. patches) to a given Python distribution defined by other tags in this extension.""", ) +_dep_tag = tag_class( + attrs = { + "extra_targets": attr.string_list( + doc = """\ +A list of extra target names in the package that are expected to be available. +See {obj}`pip.parse.extra_hub_aliases`. +""", + default = [], + ), + "name": attr.string( + doc = "The name of a pypi package. Note that the name is normalized.", + mandatory = True, + ), + }, + doc = """\ +Declare an abstract PyPI dependency to ensure its target structure exists in the unified hub. + +This is useful for targets or rules that need to depend on a package (e.g., `@pypi//numpy`) +but do not want to force a specific version or concrete requirements lock file on their +consumers. The concrete version and implementation must be provided by downstreams calling +`pip.parse`. If they are not, the target will still be defined, but it will result in an +execution-phase error when built. +""", +) + pypi = module_extension( environ = ["RULES_PYTHON_PYPI_HUB_RESERVED"], doc = """\ @@ -1065,6 +1110,7 @@ terms used in this extension. ::: """, ), + "dep": _dep_tag, "override": _override_tag, "parse": tag_class( attrs = _pip_parse_ext_attrs(), diff --git a/tests/integration/unified_pypi/BUILD.bazel b/tests/integration/unified_pypi/BUILD.bazel index 8a37d4b153..c0b9905fa6 100644 --- a/tests/integration/unified_pypi/BUILD.bazel +++ b/tests/integration/unified_pypi/BUILD.bazel @@ -46,3 +46,16 @@ py_binary( }, deps = ["@pypi//six"], ) + +py_binary( + name = "bin_declared_only", + srcs = ["bin_declared_only.py"], + deps = ["@pypi//declared_only_pkg"], +) + +py_binary( + name = "bin_declared_only_alias", + srcs = ["bin_declared_only.py"], + main = "bin_declared_only.py", + deps = ["@pypi//declared_only_pkg:declared-only-alias"], +) diff --git a/tests/integration/unified_pypi/MODULE.bazel b/tests/integration/unified_pypi/MODULE.bazel index 0d0f44f61c..6a4b87e015 100644 --- a/tests/integration/unified_pypi/MODULE.bazel +++ b/tests/integration/unified_pypi/MODULE.bazel @@ -45,4 +45,8 @@ pip.parse( use_repo(pip, "pypi_b") pip.default(default_hub = "pypi_b") +pip.dep( + name = "declared-only-pkg", + extra_targets = ["declared-only-alias"], +) use_repo(pip, "pypi") diff --git a/tests/integration/unified_pypi/bin_declared_only.py b/tests/integration/unified_pypi/bin_declared_only.py new file mode 100644 index 0000000000..0b03607e77 --- /dev/null +++ b/tests/integration/unified_pypi/bin_declared_only.py @@ -0,0 +1,2 @@ +# Dummy file for integration test +print("declared_only") diff --git a/tests/integration/unified_pypi_test.py b/tests/integration/unified_pypi_test.py index 707ab13444..1b4aee4bef 100644 --- a/tests/integration/unified_pypi_test.py +++ b/tests/integration/unified_pypi_test.py @@ -74,6 +74,27 @@ def test_invalid_default_hub_fails_evaluation(self): "default_hub 'invalid_hub' is not a defined PyPI hub", ) + def test_unimplemented_declared_dep_fails_build(self): + # Even though cquery succeeds: + self.run_bazel("cquery", "//:bin_declared_only") + + # Build must fail because the package is not implemented by any concrete hub + result = self.run_bazel("build", "//:bin_declared_only", check=False) + self.assertNotEqual(result.exit_code, 0) + self.assert_result_matches( + result, + 'ERROR: PyPI package "declared_only_pkg" is not available when building under PyPI hub "pypi_b".', + ) + + def test_unimplemented_declared_dep_alias_fails_build(self): + # Build must fail for alias too + result = self.run_bazel("build", "//:bin_declared_only_alias", check=False) + self.assertNotEqual(result.exit_code, 0) + self.assert_result_matches( + result, + 'ERROR: PyPI package "declared_only_pkg:declared-only-alias" is not available when building under PyPI hub "pypi_b".', + ) + if __name__ == "__main__": unittest.main() diff --git a/tests/pypi/extension/extension_tests.bzl b/tests/pypi/extension/extension_tests.bzl index bc4c0bcb5b..143f06a471 100644 --- a/tests/pypi/extension/extension_tests.bzl +++ b/tests/pypi/extension/extension_tests.bzl @@ -90,7 +90,13 @@ _default_tags_default = [ }.items() ] -def _mod(*, name, default = _default_tags_default, parse = [], override = [], whl_mods = [], is_root = True): +def _dep(*, name, extra_targets = []): + return struct( + name = name, + extra_targets = extra_targets, + ) + +def _mod(*, name, default = _default_tags_default, parse = [], override = [], whl_mods = [], dep = [], is_root = True): return struct( name = name, tags = struct( @@ -98,6 +104,7 @@ def _mod(*, name, default = _default_tags_default, parse = [], override = [], wh override = override, whl_mods = whl_mods, default = default, + dep = dep, ), is_root = is_root, ) @@ -106,6 +113,7 @@ def _parse_modules(env, **kwargs): return env.expect.that_struct( parse_modules(**kwargs), attrs = dict( + declared_deps = subjects.dict, default_hub = subjects.str, exposed_packages = subjects.dict, hub_group_map = subjects.dict, @@ -435,6 +443,35 @@ def _test_default_hub_precedence(env): _tests.append(_test_default_hub_precedence) +def _test_extension_dep(env): + pypi = _parse_modules( + env, + module_ctx = _pypi_mock_mctx( + _mod( + name = "my_module", + dep = [ + _dep( + name = "declared-pkg", + extra_targets = ["declared-alias"], + ), + ], + ), + os_name = "linux", + arch_name = "x86_64", + ), + available_interpreters = {}, + minor_mapping = {}, + ) + + pypi.declared_deps().contains_exactly({"declared_pkg": {"declared-alias": None}}) + pypi.exposed_packages().contains_exactly({}) + pypi.hub_group_map().contains_exactly({}) + pypi.hub_whl_map().contains_exactly({}) + pypi.whl_libraries().contains_exactly({}) + pypi.whl_mods().contains_exactly({}) + +_tests.append(_test_extension_dep) + def extension_test_suite(name): """Create the test suite.