diff --git a/.agents/rules/bzl.md b/.agents/rules/bzl.md new file mode 100644 index 0000000000..104c52c590 --- /dev/null +++ b/.agents/rules/bzl.md @@ -0,0 +1,8 @@ +--- +trigger: glob +description: Starlark / Bazel .bzl file coding style rules +globs: *.bzl +--- + +* Use triple-quoted strings for multi-line rule doc args. +* Don't use backslash line continuation in rule doc args. diff --git a/docs/pypi/download.md b/docs/pypi/download.md index 6705df1f3a..ebdb63bf5f 100644 --- a/docs/pypi/download.md +++ b/docs/pypi/download.md @@ -121,6 +121,41 @@ Shared library targets can simply depend on the unified hub (e.g., `@pypi//numpy`), and the dependency will automatically resolve to the correct wheel version from the active hub during the build. +### Declaring Abstract Dependencies (pip.dep) + +:::{versionadded} VERSION_NEXT_FEATURE +Declaring abstract PyPI dependencies via `pip.dep` tags. +::: + +Sometimes a shared library target or a ruleset needs to depend on a PyPI +package (e.g., `@pypi//numpy`), but does not want to force a specific package +version or a concrete `requirements.txt` lock file on its consumers. + +Instead of calling `pip.parse()`, the module can declare its dependency using +the `pip.dep` tag: + +```starlark +pip = use_extension("@rules_python//python/extensions:pip.bzl", "pip") + +# Declare an abstract dependency on 'numpy' and specify extra targets that +# are expected to be available in the package. +pip.dep( + name = "numpy", + extra_targets = ["extra-alias"], +) +``` + +This ensures that the target structure `@pypi//numpy` (and +`@pypi//numpy:extra-alias`) exists in the unified `@pypi` hub repository, so the +declaring module can compile and analyze successfully without needing any local +requirements file. + +The actual concrete implementation and version of the package must be provided +by a downstream module calling `pip.parse`. + +If a downstream module attempts to build a target that depends on an abstract +dependency, but has not provided a concrete implementation for it via any +`pip.parse` call, the build will fail at execution time. As with any repository rule or extension, if you would like to ensure that `pip_parse` is diff --git a/news/pip-dep-tag-class.added.md b/news/pip-dep-tag-class.added.md new file mode 100644 index 0000000000..5306186081 --- /dev/null +++ b/news/pip-dep-tag-class.added.md @@ -0,0 +1,4 @@ +(pypi) Added a `dep` tag class to the `pip` bzlmod extension. This allows +modules to declare abstract PyPI dependencies, ensuring target structures +exist in the unified hub, while allowing other modules to provide the +concrete implementation via `pip.parse`. diff --git a/python/private/pypi/extension.bzl b/python/private/pypi/extension.bzl index 62d2c8de16..a3e775e08b 100644 --- a/python/private/pypi/extension.bzl +++ b/python/private/pypi/extension.bzl @@ -424,6 +424,15 @@ You cannot use both the additive_build_content and additive_build_content_file a pip_attr = pip_attr, ) + # dict[str package, dict[str, None] extra_targets] + declared_deps = {} + for mod in module_ctx.modules: + for dep_attr in mod.tags.dep: + name = normalize_name(dep_attr.name) + targets = declared_deps.setdefault(name, {}) + for target in dep_attr.extra_targets: + targets[target] = None + # Keeps track of all the hub's whl repos across the different versions. # dict[hub, dict[whl, dict[version, str pip]]] # Where hub, whl, and pip are the repo names @@ -448,6 +457,7 @@ You cannot use both the additive_build_content and additive_build_content_file a return struct( config = config, + declared_deps = declared_deps, default_hub = config.default_hub or renamed_default_hub, exposed_packages = exposed_packages, extra_aliases = extra_aliases, @@ -492,6 +502,16 @@ def _create_unified_hub_repo(mods): if hub_name not in extra_aliases[qual_alias]: extra_aliases[qual_alias].append(hub_name) + for dep_name, extra_targets in mods.declared_deps.items(): + norm_pkg = normalize_name(dep_name) + if norm_pkg not in packages: + packages[norm_pkg] = [] + + for target_name in extra_targets: + qual_alias = "%s:%s" % (norm_pkg, target_name) + if qual_alias not in extra_aliases: + extra_aliases[qual_alias] = [] + unified_hub_repo( name = "pypi", default_hub = mods.default_hub or (hubs[0] if hubs else ""), @@ -1015,6 +1035,31 @@ Apply any overrides (e.g. patches) to a given Python distribution defined by other tags in this extension.""", ) +_dep_tag = tag_class( + attrs = { + "extra_targets": attr.string_list( + doc = """\ +A list of extra target names in the package that are expected to be available. +See {obj}`pip.parse.extra_hub_aliases`. +""", + default = [], + ), + "name": attr.string( + doc = "The name of a pypi package. Note that the name is normalized.", + mandatory = True, + ), + }, + doc = """\ +Declare an abstract PyPI dependency to ensure its target structure exists in the unified hub. + +This is useful for targets or rules that need to depend on a package (e.g., `@pypi//numpy`) +but do not want to force a specific version or concrete requirements lock file on their +consumers. The concrete version and implementation must be provided by downstreams calling +`pip.parse`. If they are not, the target will still be defined, but it will result in an +execution-phase error when built. +""", +) + pypi = module_extension( environ = ["RULES_PYTHON_PYPI_HUB_RESERVED"], doc = """\ @@ -1065,6 +1110,7 @@ terms used in this extension. ::: """, ), + "dep": _dep_tag, "override": _override_tag, "parse": tag_class( attrs = _pip_parse_ext_attrs(), diff --git a/tests/integration/unified_pypi/BUILD.bazel b/tests/integration/unified_pypi/BUILD.bazel index 8a37d4b153..c0b9905fa6 100644 --- a/tests/integration/unified_pypi/BUILD.bazel +++ b/tests/integration/unified_pypi/BUILD.bazel @@ -46,3 +46,16 @@ py_binary( }, deps = ["@pypi//six"], ) + +py_binary( + name = "bin_declared_only", + srcs = ["bin_declared_only.py"], + deps = ["@pypi//declared_only_pkg"], +) + +py_binary( + name = "bin_declared_only_alias", + srcs = ["bin_declared_only.py"], + main = "bin_declared_only.py", + deps = ["@pypi//declared_only_pkg:declared-only-alias"], +) diff --git a/tests/integration/unified_pypi/MODULE.bazel b/tests/integration/unified_pypi/MODULE.bazel index 0d0f44f61c..6a4b87e015 100644 --- a/tests/integration/unified_pypi/MODULE.bazel +++ b/tests/integration/unified_pypi/MODULE.bazel @@ -45,4 +45,8 @@ pip.parse( use_repo(pip, "pypi_b") pip.default(default_hub = "pypi_b") +pip.dep( + name = "declared-only-pkg", + extra_targets = ["declared-only-alias"], +) use_repo(pip, "pypi") diff --git a/tests/integration/unified_pypi/bin_declared_only.py b/tests/integration/unified_pypi/bin_declared_only.py new file mode 100644 index 0000000000..0b03607e77 --- /dev/null +++ b/tests/integration/unified_pypi/bin_declared_only.py @@ -0,0 +1,2 @@ +# Dummy file for integration test +print("declared_only") diff --git a/tests/integration/unified_pypi_test.py b/tests/integration/unified_pypi_test.py index 707ab13444..1b4aee4bef 100644 --- a/tests/integration/unified_pypi_test.py +++ b/tests/integration/unified_pypi_test.py @@ -74,6 +74,27 @@ def test_invalid_default_hub_fails_evaluation(self): "default_hub 'invalid_hub' is not a defined PyPI hub", ) + def test_unimplemented_declared_dep_fails_build(self): + # Even though cquery succeeds: + self.run_bazel("cquery", "//:bin_declared_only") + + # Build must fail because the package is not implemented by any concrete hub + result = self.run_bazel("build", "//:bin_declared_only", check=False) + self.assertNotEqual(result.exit_code, 0) + self.assert_result_matches( + result, + 'ERROR: PyPI package "declared_only_pkg" is not available when building under PyPI hub "pypi_b".', + ) + + def test_unimplemented_declared_dep_alias_fails_build(self): + # Build must fail for alias too + result = self.run_bazel("build", "//:bin_declared_only_alias", check=False) + self.assertNotEqual(result.exit_code, 0) + self.assert_result_matches( + result, + 'ERROR: PyPI package "declared_only_pkg:declared-only-alias" is not available when building under PyPI hub "pypi_b".', + ) + if __name__ == "__main__": unittest.main() diff --git a/tests/pypi/extension/extension_tests.bzl b/tests/pypi/extension/extension_tests.bzl index bc4c0bcb5b..143f06a471 100644 --- a/tests/pypi/extension/extension_tests.bzl +++ b/tests/pypi/extension/extension_tests.bzl @@ -90,7 +90,13 @@ _default_tags_default = [ }.items() ] -def _mod(*, name, default = _default_tags_default, parse = [], override = [], whl_mods = [], is_root = True): +def _dep(*, name, extra_targets = []): + return struct( + name = name, + extra_targets = extra_targets, + ) + +def _mod(*, name, default = _default_tags_default, parse = [], override = [], whl_mods = [], dep = [], is_root = True): return struct( name = name, tags = struct( @@ -98,6 +104,7 @@ def _mod(*, name, default = _default_tags_default, parse = [], override = [], wh override = override, whl_mods = whl_mods, default = default, + dep = dep, ), is_root = is_root, ) @@ -106,6 +113,7 @@ def _parse_modules(env, **kwargs): return env.expect.that_struct( parse_modules(**kwargs), attrs = dict( + declared_deps = subjects.dict, default_hub = subjects.str, exposed_packages = subjects.dict, hub_group_map = subjects.dict, @@ -435,6 +443,35 @@ def _test_default_hub_precedence(env): _tests.append(_test_default_hub_precedence) +def _test_extension_dep(env): + pypi = _parse_modules( + env, + module_ctx = _pypi_mock_mctx( + _mod( + name = "my_module", + dep = [ + _dep( + name = "declared-pkg", + extra_targets = ["declared-alias"], + ), + ], + ), + os_name = "linux", + arch_name = "x86_64", + ), + available_interpreters = {}, + minor_mapping = {}, + ) + + pypi.declared_deps().contains_exactly({"declared_pkg": {"declared-alias": None}}) + pypi.exposed_packages().contains_exactly({}) + pypi.hub_group_map().contains_exactly({}) + pypi.hub_whl_map().contains_exactly({}) + pypi.whl_libraries().contains_exactly({}) + pypi.whl_mods().contains_exactly({}) + +_tests.append(_test_extension_dep) + def extension_test_suite(name): """Create the test suite.