diff --git a/commodore/cluster.py b/commodore/cluster.py index 8900ebb20..b008376dd 100644 --- a/commodore/cluster.py +++ b/commodore/cluster.py @@ -5,7 +5,7 @@ import textwrap from datetime import datetime -from typing import Any, Optional, Union +from typing import Any, Iterable, Optional, Union import click @@ -200,6 +200,7 @@ def render_target( target: str, components: dict[str, Component], component: Optional[str] = None, + extra_classes: Optional[Iterable[str]] = None, ): if not component: component = target @@ -221,6 +222,8 @@ def render_target( click.secho(f" > Default file for class {c} missing", fg="yellow") classes.append("global.commodore") + if extra_classes: + classes.extend(extra_classes) if not bootstrap: if not inv.component_file(target).is_file(): @@ -232,12 +235,21 @@ def render_target( return generate_target(inv, target, components, classes, component) -def update_target(cfg: Config, target: str, component: Optional[str] = None): +def update_target( + cfg: Config, + target: str, + component: Optional[str] = None, + extra_classes: Optional[Iterable[str]] = None, +): click.secho(f"Updating Kapitan target for {target}...", bold=True) file = cfg.inventory.target_file(target) os.makedirs(file.parent, exist_ok=True) targetdata = render_target( - cfg.inventory, target, cfg.get_components(), component=component + cfg.inventory, + target, + cfg.get_components(), + component=component, + extra_classes=extra_classes, ) yaml_dump(targetdata, file) diff --git a/commodore/compile.py b/commodore/compile.py index 67b4f78b7..e4e89372d 100644 --- a/commodore/compile.py +++ b/commodore/compile.py @@ -25,6 +25,7 @@ verify_version_overrides, ) from .dependency_mgmt.component_library import create_component_library_aliases +from .dependency_mgmt.component_dependency import validate_catalog_dependencies from .dependency_mgmt.jsonnet_bundler import ( fetch_jsonnet_libraries, jsonnet_dependencies, @@ -241,6 +242,9 @@ def setup_compile_environment(config: Config) -> tuple[dict[str, Any], Iterable[ # Raise exception if component version override without URL is present in the # hierarchy. verify_version_overrides(cluster_parameters, config.get_component_aliases()) + # Raise exception if the catalog violates any component dependency version + # requirements. + validate_catalog_dependencies(config, inventory) for component in config.get_components().values(): ckey = component.parameters_key diff --git a/commodore/component/compile.py b/commodore/component/compile.py index c755a98d2..3a096ca19 100644 --- a/commodore/component/compile.py +++ b/commodore/component/compile.py @@ -6,19 +6,27 @@ from collections.abc import Iterable from pathlib import Path as P from textwrap import dedent -from typing import Optional +from typing import Any, Optional import click import git -from commodore.cluster import generate_target +from commodore.cluster import update_target from commodore.config import Config from commodore.component import Component +from commodore.dependency_mgmt import fetch_components, create_component_symlinks +from commodore.dependency_mgmt.component_dependency import ( + collect_catalog_dependencies, + ComponentDependency, +) from commodore.dependency_mgmt.component_library import ( validate_component_library_name, create_component_library_aliases, ) -from commodore.dependency_mgmt.jsonnet_bundler import fetch_jsonnet_libraries +from commodore.dependency_mgmt.jsonnet_bundler import ( + fetch_jsonnet_libraries, + jsonnet_dependencies, +) from commodore.helpers import kapitan_inventory, kapitan_compile, relsymlink, yaml_dump from commodore.inventory import Inventory from commodore.inventory.lint import check_removed_reclass_variables @@ -40,7 +48,6 @@ def compile_component( component_path = P(component_path_).resolve() value_files = [P(f).resolve() for f in value_files_] search_paths = [P(d).resolve() for d in search_paths_] - search_paths.append(component_path / "vendor") output_path = P(output_path_).resolve() if not component_name: @@ -59,12 +66,15 @@ def compile_component( ) temp_dir = P(tempfile.mkdtemp(prefix="component-")).resolve() + search_paths.append(temp_dir / "vendor") config.work_dir = temp_dir try: if config.debug: click.echo(f" > Created temp workspace: {config.work_dir}") inv = config.inventory inv.ensure_dirs() + inv.global_config_dir.mkdir() + yaml_dump({}, inv.global_config_dir / "commodore.yml") search_paths.append(inv.dependencies_dir) component = _setup_component( config, @@ -72,7 +82,8 @@ def compile_component( instance_name, component_path, ) - _prepare_kapitan_inventory(inv, component, value_files, instance_name) + config.register_component(component) + _prepare_kapitan_inventory(config, component, value_files, instance_name) # Raise error if component uses removed reclass parameters check_removed_reclass_variables( @@ -81,21 +92,50 @@ def compile_component( [component.defaults_file, component.class_file] + value_files, ) - # Verify component alias + # Fetch and install component dependencies + click.secho( + f"Discovering component dependencies for {instance_name}...", bold=True + ) nodes = kapitan_inventory(config) + component_deps = collect_catalog_dependencies(config, nodes) + if len(component_deps) > 0: + component_deps["argocd"] = ComponentDependency.parse( + component.name, + "argocd", + {"url": "https://github.com/projectsyn/component-argocd.git"}, + ) + + _setup_dependencies(inv, component_deps) + update_target(config, inv.bootstrap_target) + + fetch_components(config) + + update_target(config, inv.bootstrap_target) + _prepare_kapitan_inventory(config, component, value_files, instance_name) + + cluster_parameters = kapitan_inventory(config)[inv.bootstrap_target][ + "parameters" + ] + nodes = kapitan_inventory(config) + else: + _setup_fake_argocd_lib(inv) + cluster_parameters = kapitan_inventory(config)[instance_name]["parameters"] + create_component_symlinks(config, component) + search_paths.append(component_path / "vendor") + + # Fetch Jsonnet dependencies + for component in config.get_components().values(): + ckey = component.parameters_key + component.render_jsonnetfile_json(cluster_parameters[ckey]) + + fetch_jsonnet_libraries(config.work_dir, deps=jsonnet_dependencies(config)) + + # Verify component alias config.verify_component_aliases(nodes, bootstrap_target=instance_name) cluster_params = nodes[instance_name]["parameters"] create_component_library_aliases(config, cluster_params) - # Render jsonnetfile.jsonnet if necessary - component_params = nodes[instance_name]["parameters"].get( - component_name.replace("-", "_"), {} - ) - component.render_jsonnetfile_json(component_params) - # Fetch Jsonnet libs - fetch_jsonnet_libraries(component_path) - # Compile component kapitan_compile( config, @@ -180,7 +220,10 @@ def _setup_component( def _prepare_kapitan_inventory( - inv: Inventory, component: Component, value_files: Iterable[P], instance_name: str + config: Config, + component: Component, + value_files: Iterable[P], + instance_name: str, ): """ Setup Kapitan inventory. @@ -188,6 +231,9 @@ def _prepare_kapitan_inventory( Create component symlinks, values file symlinks, setup params class with fake values and Kapitan target for the component, create a fake `lib/argocd.libjsonnet`. """ + + inv = config.inventory + component_class_file = component.class_file component_defaults_file = component.defaults_file if not component_class_file.exists(): @@ -200,12 +246,14 @@ def _prepare_kapitan_inventory( ) # Create class symlink - relsymlink(component_class_file, inv.components_dir) + relsymlink( + component_class_file, inv.components_dir, dest_name=f"{instance_name}.yml" + ) # Create defaults symlink relsymlink( component_defaults_file, inv.defaults_dir, - dest_name=f"{component.name}.yml", + dest_name=f"{instance_name}.yml", ) # Create component symlink relsymlink(component.target_directory, inv.dependencies_dir, component.name) @@ -229,9 +277,6 @@ def _prepare_kapitan_inventory( "cloud": "cloudscale", "region": "rma1", }, - "argocd": { - "namespace": "test", - }, "components": { component.name: { "url": f"https://example.com/{component.name}.git", @@ -251,18 +296,10 @@ def _prepare_kapitan_inventory( # Create test target value_classes = [f"{c.stem}" for c in value_files] - classes = [ - f"params.{inv.bootstrap_target}", - f"defaults.{component.name}", - f"components.{component.name}", - ] + value_classes - yaml_dump( - generate_target( - inv, instance_name, {component.name: component}, classes, component.name - ), - inv.target_file(instance_name), - ) + update_target(config, instance_name, component.name, value_classes) + +def _setup_fake_argocd_lib(inv: Inventory): # Fake Argo CD lib # We plug "fake" Argo CD library here because every component relies on it # and we don't want to provide it every time when compiling a single component. @@ -275,3 +312,13 @@ def _prepare_kapitan_inventory( App: ArgoApp, Project: ArgoProject, }""")) + + +def _setup_dependencies(inv: Inventory, dependencies: dict[str, ComponentDependency]): + dependencies_yaml: dict[str, Any] = { + "applications": list(dependencies.keys()), + "parameters": { + "components": {dn: dep.component_entry for dn, dep in dependencies.items()} + }, + } + yaml_dump(dependencies_yaml, inv.global_config_dir / "commodore.yml") diff --git a/commodore/config.py b/commodore/config.py index b1b5a2030..7ab2b6e02 100644 --- a/commodore/config.py +++ b/commodore/config.py @@ -344,7 +344,7 @@ def inventory(self): def update_verbosity(self, verbose): self._verbose += verbose - def get_components(self): + def get_components(self) -> dict[str, Component]: return self._components def register_component(self, component: Component): @@ -396,7 +396,7 @@ def register_dependency_repo(self, repo_url: str) -> MultiDependency: dep.url = repo_url return dep - def get_component_aliases(self): + def get_component_aliases(self) -> dict[str, str]: return self._component_aliases def register_component_aliases(self, aliases: dict[str, str]): diff --git a/commodore/dependency_mgmt/__init__.py b/commodore/dependency_mgmt/__init__.py index e303dbe88..62b0ffef1 100644 --- a/commodore/dependency_mgmt/__init__.py +++ b/commodore/dependency_mgmt/__init__.py @@ -2,7 +2,7 @@ import itertools from concurrent.futures import ThreadPoolExecutor -from typing import Callable, Iterable +from typing import Callable, Iterable, Optional import click from click import ClickException @@ -77,7 +77,7 @@ def create_package_symlink(cfg, pname: str, package: Package): relsymlink(package.target_dir, cfg.inventory.classes_dir, dest_name=pname) -def fetch_components(cfg: Config): +def fetch_components(cfg: Config, bootstrap_target: Optional[str] = None): """ Download all components required by target. @@ -90,7 +90,7 @@ def fetch_components(cfg: Config): component_names, component_aliases = _discover_components(cfg) click.secho("Registering component aliases...", bold=True) cfg.register_component_aliases(component_aliases) - cspecs = _read_components(cfg, component_aliases) + cspecs = _read_components(cfg, component_aliases, bootstrap_target) click.secho("Fetching components...", bold=True) deps: dict[str, list] = {} @@ -184,7 +184,7 @@ def do_parallel(fun: Callable[[Config, Iterable], None], cfg: Config, data: Iter list(exe.map(fun, itertools.repeat(cfg), data)) -def register_components(cfg: Config): +def register_components(cfg: Config, bootstrap_target: Optional[str] = None): """ Discover components in the inventory, and register them if the corresponding directory in `dependencies/` exists. @@ -194,7 +194,7 @@ def register_components(cfg: Config): click.secho("Discovering included components...", bold=True) try: components, component_aliases = _discover_components(cfg) - cspecs = _read_components(cfg, component_aliases) + cspecs = _read_components(cfg, component_aliases, bootstrap_target) except KeyError as e: raise click.ClickException(f"While discovering components: {e}") click.secho("Registering components and aliases...", bold=True) diff --git a/commodore/dependency_mgmt/component_dependency.py b/commodore/dependency_mgmt/component_dependency.py new file mode 100644 index 000000000..7f9e278f5 --- /dev/null +++ b/commodore/dependency_mgmt/component_dependency.py @@ -0,0 +1,234 @@ +from __future__ import annotations + +from dataclasses import dataclass +from typing import Any, Optional + +import click +import semver + +from cel_expr_python import cel # type: ignore + +from commodore.config import Config +from commodore.component import component_parameters_key + + +class ComponentDependencyParseError(ValueError): + field: str + reason: Optional[str] + + def __init__(self, field: str, reason: Optional[str] = ""): + msg = "Error parsing dependency specification" + if reason: + msg += f": {reason}" + super().__init__(msg) + self.field = field + self.reason = reason + + +@dataclass +class ComponentDependency: + """Class for parsed component dependency specification""" + + name: str + instances: list[str] + url: str + path: Optional[str] + minverspec: Optional[str] + minversion: Optional[semver.Version] + mandatory: bool + requiredif: list[str] + + @classmethod + def parse( + cls, cname: str, depname: str, depspec: dict[str, str] + ) -> ComponentDependency: + if "url" not in depspec: + raise ComponentDependencyParseError("url", "field 'url' missing") + url = depspec["url"] + minverspec = None + if "minversion" in depspec: + minverspec = depspec["minversion"] + try: + if minverspec.startswith("v"): + minversion = semver.Version.parse(minverspec[1:]) + else: + minversion = semver.Version.parse(minverspec) + except ValueError as e: + raise ComponentDependencyParseError("minversion", str(e)) + else: + minversion = None + + # NOTE(sg): if requiredif isn't set, the dependency is mandatory + mandatory = "requiredif" not in depspec + requiredif = [] + if not mandatory: + requiredif = [depspec["requiredif"]] + + return ComponentDependency( + depname, + [cname], + url, + depspec.get("path"), + minverspec, + minversion, + mandatory, + requiredif, + ) + + def update(self, other: ComponentDependency): + if self.name != other.name: + raise ValueError( + f"Cannot merge ComponentDependency objects with different names: {self.name}, {other.name}" + ) + if self.url != other.url: + raise ValueError( + f"Cannot merge ComponentDependency objects with same name but different URLs: {self.url}, {other.url}" + ) + + if self.path != other.path: + raise ValueError( + "Cannot merge ComponentDependency objects with same name and URL but different sub-paths: " + + f"{self.path}, {other.path}" + ) + + if self.minversion and other.minversion: + cur_self = self.minversion + self.minversion = max(self.minversion, other.minversion) + if cur_self != self.minversion: + self.minverspec = other.minverspec + elif other.minversion: + self.minversion = other.minversion + self.minverspec = other.minverspec + + self.mandatory = self.mandatory or other.mandatory + self.requiredif.extend(other.requiredif) + self.instances.extend(other.instances) + + def _error_helper(self) -> tuple[str, str]: + if len(self.instances) == 1: + instances = "instance" + require = "requires" + else: + instances = "instances" + require = "require" + + return (instances, require) + + def missing_dependency_error(self) -> str: + instances, require = self._error_helper() + instances_list = ", ".join(map(lambda i: f"'{i}'", self.instances)) + return ( + f"Component {instances} {instances_list} {require} dependency " + + f"'{self.name}' which isn't present in catalog" + ) + + def not_minversion_error(self, cv: str) -> str: + instances, require = self._error_helper() + instances_list = ", ".join(map(lambda i: f"'{i}'", self.instances)) + return ( + f"Component {instances} {instances_list} {require} dependency '{self.name}' " + + f"in a version '>= {self.minverspec}': catalog has '{cv}'" + ) + + def required_for_catalog( + self, config: Config, cparams: dict[str, Any], facts: dict[str, Any] + ) -> bool: + if self.mandatory: + return True + + required = False + cel_env = cel.NewEnv(variables={"config": cel.Type.MAP, "facts": cel.Type.MAP}) + for expr in self.requiredif: + if config.debug: + click.echo(f" > Evaluating CEL expression: {expr}") + cel_expr = cel_env.compile(expr) + res = cel_expr.eval(data={"config": cparams, "facts": facts}) + if res.type() == cel.Type.ERROR: + raise ValueError( + f"Evaluation failed for `requiredif` CEL expression: {res.value()}" + ) + if res.type() != cel.Type.BOOL: + raise ValueError( + "Component dependency `requiredif` CEL expression must evaluate to a boolean" + ) + resval = res.value() + required = required or resval + + return required + + @property + def component_entry(self) -> dict[str, str]: + entry = { + "url": self.url, + "version": self.minverspec or "master", + } + if self.path: + entry["path"] = self.path + return entry + + +def collect_catalog_dependencies( + config: Config, inventory: dict[str, Any] +) -> dict[str, ComponentDependency]: + catalog_deps: dict[str, ComponentDependency] = {} + for instance, cn in config.get_component_aliases().items(): + if config.debug: + click.echo(f" > Collecting dependencies for component instance {instance}") + params = inventory[instance]["parameters"] + deps = map( + lambda d: ComponentDependency.parse(instance, *d), + params.get("commodore", {}).get("dependencies", {}).items(), + ) + + for dep in deps: + if not dep.required_for_catalog( + config, params[component_parameters_key(cn)], params["facts"] + ): + if config.debug: + click.echo(f" > Dependency {dep.name} not required for catalog") + continue + + if dep.name in catalog_deps: + catalog_deps[dep.name].update(dep) + else: + catalog_deps[dep.name] = dep + + return catalog_deps + + +def validate_catalog_dependencies(config: Config, inventory: dict[str, Any]): + click.secho("Validating component dependencies...", bold=True) + catalog_deps = collect_catalog_dependencies(config, inventory) + deperrs = [] + for dn, dep in catalog_deps.items(): + if config.verbose: + click.echo(f" > Validating dependency {dn}") + d = config.get_components().get(dn) + if not d: + deperrs.append(dep.missing_dependency_error()) + continue + + if dep.minversion: + try: + if not d.version: + raise ValueError("component instance {dn} missing version") + + if d.version.startswith("v"): + dv = semver.Version.parse(d.version[1:]) + else: + dv = semver.Version.parse(d.version) + except ValueError: + if config.verbose: + click.echo( + f" > Dependency '{dn}' present in catalog with version '{d.version}' " + + f"which doesn't parse as SemVer: assuming '{d.version} >= {dep.minversion}'" + ) + continue + if dep.minversion > dv: + deperrs.append(dep.not_minversion_error(d.version)) + + if len(deperrs) > 0: + deperrs_str = "\n * ".join(deperrs) + raise click.ClickException( + f"catalog dependency validation failed:\n * {deperrs_str}" + ) diff --git a/commodore/dependency_mgmt/version_parsing.py b/commodore/dependency_mgmt/version_parsing.py index 5318a901f..5252a08e5 100644 --- a/commodore/dependency_mgmt/version_parsing.py +++ b/commodore/dependency_mgmt/version_parsing.py @@ -69,14 +69,17 @@ def _read_versions( ignore_class_notfound: bool = False, aliases: dict[str, str] = {}, fallback: dict[str, DependencySpec] = {}, + bootstrap_target: Optional[str] = None, ) -> dict[str, DependencySpec]: deps_key = dependency_type.value deptype_str = dependency_type.name.lower() deptype_cap = deptype_str.capitalize() dependencies = {} + real_bootstrap_target = bootstrap_target or cfg.inventory.bootstrap_target + inv = kapitan_inventory(cfg, ignore_class_notfound=ignore_class_notfound) - cluster_inventory = inv[cfg.inventory.bootstrap_target] + cluster_inventory = inv[real_bootstrap_target] deps = cluster_inventory["parameters"].get(deps_key, None) if not deps: if require_key: @@ -120,18 +123,23 @@ def _read_versions( def _read_components( - cfg: Config, component_aliases: dict[str, str] + cfg: Config, + component_aliases: dict[str, str], + bootstrap_target: Optional[str] = None, ) -> dict[str, DependencySpec]: component_names = set(component_aliases.values()) alias_names = set(component_aliases.keys()) - component_names - component_versions = _read_versions(cfg, DepType.COMPONENT, component_names) + component_versions = _read_versions( + cfg, DepType.COMPONENT, component_names, bootstrap_target=bootstrap_target + ) alias_versions = _read_versions( cfg, DepType.COMPONENT, alias_names, aliases=component_aliases, fallback=component_versions, + bootstrap_target=bootstrap_target, ) for alias, aspec in alias_versions.items(): diff --git a/poetry.lock b/poetry.lock index efd7349c5..72e42d123 100644 --- a/poetry.lock +++ b/poetry.lock @@ -202,18 +202,18 @@ uvloop = ["uvloop (>=0.15.2) ; sys_platform != \"win32\"", "winloop (>=0.5.0) ; [[package]] name = "boto3" -version = "1.43.73" +version = "1.43.74" description = "The AWS SDK for Python" optional = false python-versions = ">=3.10" groups = ["main"] files = [ - {file = "boto3-1.43.73-py3-none-any.whl", hash = "sha256:5b54da301c387abe30c5b3a5335652f1ebd73e814c725ba805d27a2d477ce547"}, - {file = "boto3-1.43.73.tar.gz", hash = "sha256:6e6c755e5039f204882c01d7936a46abce539b3f5bbb534a23a3f2afbc86f576"}, + {file = "boto3-1.43.74-py3-none-any.whl", hash = "sha256:d7f6a921efbda61ed787b403df8b64a95bc66e66299e9772fc40c235c28a5bde"}, + {file = "boto3-1.43.74.tar.gz", hash = "sha256:9d4d0dcf81c941cfe7d927486435521e0f3bf657cb68b10dfe445cfe311f8839"}, ] [package.dependencies] -botocore = ">=1.43.73,<1.44.0" +botocore = ">=1.43.74,<1.44.0" jmespath = ">=0.7.1,<2.0.0" s3transfer = ">=0.19.0,<0.20.0" @@ -222,14 +222,14 @@ crt = ["botocore[crt] (>=1.21.0,<2.0a0)"] [[package]] name = "botocore" -version = "1.43.73" +version = "1.43.74" description = "Low-level, data-driven core of boto 3." optional = false python-versions = ">=3.10" groups = ["main"] files = [ - {file = "botocore-1.43.73-py3-none-any.whl", hash = "sha256:068433028e011ccbeab1dd7c46b1090c24e378397693c66e67ca571176498daa"}, - {file = "botocore-1.43.73.tar.gz", hash = "sha256:0fa1e63c24b3531be3e1bc1687a88b3be9e63a430153f24edd93efc162bb1c51"}, + {file = "botocore-1.43.74-py3-none-any.whl", hash = "sha256:5793d9170b2b9f3194fb6f667feed8986eb81f767528d52ceaf45df2242d56e9"}, + {file = "botocore-1.43.74.tar.gz", hash = "sha256:b127b24d49532b5d7b0851b4abb3d3b99f58a121f87f07c23dac7c4d88b25c72"}, ] [package.dependencies] @@ -252,6 +252,36 @@ files = [ {file = "cachetools-7.1.7.tar.gz", hash = "sha256:a3e2a00b14d8f8a6b70c1dae7b4685e7ad3bc965c5b42124a2d6ce895da6cf50"}, ] +[[package]] +name = "cel-expr-python" +version = "0.1.3" +description = "The CEL Python runtime" +optional = false +python-versions = ">=3.11" +groups = ["main"] +files = [ + {file = "cel_expr_python-0.1.3-cp311-cp311-macosx_10_13_x86_64.whl", hash = "sha256:5b2ecfc4f4e8120928b446262b13977ee3326b535c1684bdc3eb2fb990298dba"}, + {file = "cel_expr_python-0.1.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:11e4cf0a3b2bf7b4231df4ed1f359de006001f2680676c99631df86744f9c1f2"}, + {file = "cel_expr_python-0.1.3-cp311-cp311-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:cc1d80c53affcaf7f31b09dd3e10d87add48833b1de51f037b8fcfb549e7362f"}, + {file = "cel_expr_python-0.1.3-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:292ae505727bfcfc1986c8bf653c2580faa06f46af56b9087060eafdf2a08652"}, + {file = "cel_expr_python-0.1.3-cp311-cp311-win_amd64.whl", hash = "sha256:8b56ebf586df6ec4dae0449d90bd9f6da0027bef407f2ffbb92701a9886ee43c"}, + {file = "cel_expr_python-0.1.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:5ab17a336e7f01e3868fc454630c05596fb8a16293833fa34a7a2f946faa835f"}, + {file = "cel_expr_python-0.1.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:4e2694afe68a0484ee7c99a79887309511f6738e562a5d2c4ff566dc791e7ec6"}, + {file = "cel_expr_python-0.1.3-cp312-cp312-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4170d1e6a3cf359b0ec7c1fb50317daa3e3e3eb12b61311260714a1e02313fbe"}, + {file = "cel_expr_python-0.1.3-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ab3b0a418b3911177a77ee69fef1f4199b4a053e2ddd1c339119e58c9cd72824"}, + {file = "cel_expr_python-0.1.3-cp312-cp312-win_amd64.whl", hash = "sha256:925bbf323fdf2743491ddf411e40b5c4a12ab102e3bc7e0bfec0fed1c6defa3b"}, + {file = "cel_expr_python-0.1.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:ac60fdb981d1435faadc536c0582ada321a3a392c18ba70e94e3f3caa0f37437"}, + {file = "cel_expr_python-0.1.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:b8eb8d3c92b11e99fffddb846ba68f8b1ba12a3bf1bc0ccdd8c2914c7458f3a4"}, + {file = "cel_expr_python-0.1.3-cp313-cp313-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ea4679e74567c145c824e1ef049ae340af5b2a1a4451324d215e58ed5db20969"}, + {file = "cel_expr_python-0.1.3-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:aa80370ed20cee201a7671634f9c58dc17fd93f15addf680d7e0393969bc51b3"}, + {file = "cel_expr_python-0.1.3-cp313-cp313-win_amd64.whl", hash = "sha256:cd6ff40b4a89af6b62ca24d431893d8fcce5cca6149f06926a0e88211a370645"}, + {file = "cel_expr_python-0.1.3-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:665fbb261c4733e69a4022763c16f7655cfe6e057cd394ad9f36dfee92b9df8c"}, + {file = "cel_expr_python-0.1.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:b3e1ba34748796624b2cf8c9bc664b79baccc0662c97299a1991278ef45d3aed"}, + {file = "cel_expr_python-0.1.3-cp314-cp314-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0d2035831cfb9c0ae36c91e3de421828f8fa27173680e86676a7d6e0f68300a0"}, + {file = "cel_expr_python-0.1.3-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9b36c7a36ca865b147e1fc68f3f6faaae5b113e9de4d77e7eb751c95e1547a99"}, + {file = "cel_expr_python-0.1.3-cp314-cp314-win_amd64.whl", hash = "sha256:4711f59f0dd3fcabf68a617685ab2857ed931cc10e2f1595abe7d4d99f921bba"}, +] + [[package]] name = "certifi" version = "2026.7.22" @@ -3063,6 +3093,18 @@ botocore = ">=1.37.4,<2.0a0" [package.extras] crt = ["botocore[crt] (>=1.37.4,<2.0a0)"] +[[package]] +name = "semver" +version = "3.0.4" +description = "Python helper for Semantic Versioning (https://semver.org)" +optional = false +python-versions = ">=3.7" +groups = ["main"] +files = [ + {file = "semver-3.0.4-py3-none-any.whl", hash = "sha256:9c824d87ba7f7ab4a1890799cec8596f15c1241cb473404ea1cb0c55e4b04746"}, + {file = "semver-3.0.4.tar.gz", hash = "sha256:afc7d8c584a5ed0a11033af086e8af226a9c0b206f313e0301f8dd7b6b589602"}, +] + [[package]] name = "shellingham" version = "1.5.4" @@ -3349,4 +3391,4 @@ dev = ["doc8", "flake8", "flake8-import-order", "rstcheck[sphinx]", "ruff", "sph [metadata] lock-version = "2.1" python-versions = ">=3.11, <3.15" -content-hash = "5a0dbe7c2aa1f21063ffb3609804a16cfd9ceb71736615997d95d0dc61d41232" +content-hash = "c303c966b0fbccae2134500d96b3a4a871c45b7398855ce7616bb590259fcd36" diff --git a/pyproject.toml b/pyproject.toml index 91d0ab6ee..e79f91cb7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -53,6 +53,8 @@ PyGithub = "2.9.1" reclass-rs = "0.11.0" gojsonnet = "0.22.0" pygorpmrustinfo = "0.1.5" +semver = "3.0.4" +cel-expr-python = "^0.1.3" [tool.poetry.group.dev.dependencies] tox = "3.28.0" diff --git a/tests/test_dependency_mgmt_component_dependency.py b/tests/test_dependency_mgmt_component_dependency.py new file mode 100644 index 000000000..19583affe --- /dev/null +++ b/tests/test_dependency_mgmt_component_dependency.py @@ -0,0 +1,477 @@ +import copy + +from pathlib import Path +from typing import Optional + +import pytest +import semver + +from commodore.config import Config +from commodore.component import Component +from commodore.multi_dependency import MultiDependency + +from commodore.dependency_mgmt import component_dependency + + +def _make_dep( + name: str, + url: str, + instances: list[str] = ["test-component"], + path: Optional[str] = None, + minverspec: Optional[str] = None, + requiredif: Optional[list[str]] = None, +): + mandatory = False + if not requiredif: + mandatory = True + minversion = None + if minverspec: + try: + minversion = semver.Version.parse(minverspec) + except ValueError: + minversion = semver.Version.parse(minverspec[1:]) + + return component_dependency.ComponentDependency( + name, instances, url, path, minverspec, minversion, mandatory, requiredif or [] + ) + + +@pytest.mark.parametrize( + "depname,depspec,expected", + [ + ( + "argocd", + {"url": "https://github.com/projectsyn/component-argocd.git"}, + _make_dep("argocd", "https://github.com/projectsyn/component-argocd.git"), + ), + ( + "argocd", + { + "url": "https://github.com/projectsyn/component-argocd.git", + "minversion": "v1.0.0", + }, + _make_dep( + "argocd", + "https://github.com/projectsyn/component-argocd.git", + minverspec="v1.0.0", + ), + ), + ( + "argocd", + { + "url": "https://github.com/projectsyn/component-argocd.git", + "requiredif": "facts.distribution == 'openshift4'", + }, + _make_dep( + "argocd", + "https://github.com/projectsyn/component-argocd.git", + requiredif=["facts.distribution == 'openshift4'"], + ), + ), + ], +) +def test_component_dependency_parse( + depname: str, + depspec: dict[str, str], + expected: component_dependency.ComponentDependency, +): + dep = component_dependency.ComponentDependency.parse( + "test-component", depname, depspec + ) + assert dep == expected + + +@pytest.mark.parametrize( + "depname,depspec,expected_error", + [ + ("argocd", {}, "Error parsing dependency specification: field 'url' missing"), + ( + "argocd", + { + "url": "https://github.com/projectsyn/component-argocd.git", + "minversion": "foo", + }, + "Error parsing dependency specification: foo is not valid SemVer string", + ), + ], +) +def test_component_dependency_parse_error( + depname: str, depspec: dict[str, str], expected_error: str +): + + with pytest.raises(component_dependency.ComponentDependencyParseError) as e: + component_dependency.ComponentDependency.parse( + "test-component", depname, depspec + ) + + assert expected_error in str(e) + + +@pytest.mark.parametrize( + "d1_min,d2_min,expected_min", + [ + (None, None, None), + (None, "v1.0.0", "v1.0.0"), + ("v1.2.3", "v1.0.0", "v1.2.3"), + ("v1.0.0", "v1.2.3", "v1.2.3"), + ], +) +def test_component_dependency_update( + d1_min: Optional[str], d2_min: Optional[str], expected_min: Optional[str] +): + d1 = _make_dep( + "argocd", + "https://github.com/projectsyn/component-argocd.git", + instances=["cilium"], + minverspec=d1_min, + ) + d2 = _make_dep( + "argocd", + "https://github.com/projectsyn/component-argocd.git", + instances=["rook-ceph"], + minverspec=d2_min, + ) + d1.update(d2) + assert d1.minverspec == expected_min + if expected_min: + assert d1.minversion == semver.Version.parse(expected_min[1:]) + assert d1.instances == ["cilium", "rook-ceph"] + + +@pytest.mark.parametrize( + "d1,d2,expected_error", + [ + ( + {"name": "argocd"}, + {"name": "foo"}, + "Cannot merge ComponentDependency objects with different names: argocd, foo", + ), + ( + { + "name": "argocd", + "url": "https://github.com/projectsyn/component-argocd.git", + }, + { + "name": "argocd", + "url": "https://github.com/projectsyn/component-argocd", + }, + "Cannot merge ComponentDependency objects with same name but different URLs: " + + "https://github.com/projectsyn/component-argocd.git, https://github.com/projectsyn/component-argocd", + ), + ( + { + "name": "argocd", + "url": "https://github.com/projectsyn/component-argocd.git", + }, + { + "name": "argocd", + "url": "https://github.com/projectsyn/component-argocd.git", + "path": "foobar", + }, + "Cannot merge ComponentDependency objects with same name and URL but different sub-paths: " + + "None, foobar", + ), + ], +) +def test_component_dependency_update_error( + d1: dict[str, str], d2: dict[str, str], expected_error: str +): + d1 = _make_dep( + d1["name"], d1.get("url", ""), instances=["cilium"], path=d1.get("path") + ) + d2 = _make_dep( + d2["name"], d2.get("url", ""), instances=["rook-ceph"], path=d2.get("path") + ) + + with pytest.raises(ValueError) as e: + d1.update(d2) + + assert expected_error in str(e) + + +def test_component_dependency_error_helpers_single_instance(): + d = _make_dep( + "argocd", + "https://github.com/projectsyn/component-argocd.git", + minverspec="v1.0.0", + ) + assert ( + d.missing_dependency_error() + == "Component instance 'test-component' requires dependency 'argocd' which isn't present in catalog" + ) + assert ( + d.not_minversion_error("v0.8.1") + == "Component instance 'test-component' requires dependency 'argocd' in a version '>= v1.0.0': " + + "catalog has 'v0.8.1'" + ) + + +def test_component_dependency_error_helpers_multiple_instances(): + d = _make_dep( + "argocd", + "https://github.com/projectsyn/component-argocd.git", + minverspec="v1.0.0", + instances=["test-component-1", "test-component-2"], + ) + assert ( + d.missing_dependency_error() + == "Component instances 'test-component-1', 'test-component-2' require dependency 'argocd' " + + "which isn't present in catalog" + ) + assert ( + d.not_minversion_error("v0.8.1") + == "Component instances 'test-component-1', 'test-component-2' require dependency 'argocd' " + + "in a version '>= v1.0.0': catalog has 'v0.8.1'" + ) + + +def test_component_dependency_mandatory_required_for_catalog(config: Config): + d = _make_dep( + "argocd", + "https://github.com/projectsyn/component-argocd.git", + ) + assert d.required_for_catalog(config, {}, {}) + + +@pytest.mark.parametrize( + "requiredif,required", + [ + (["facts.distribution=='openshift4'"], False), + (["facts.distribution=='talos'"], True), + ( + [ + "facts.distribution=='talos' && (config.foo=='bar' || config.param == 'temporary')" + ], + True, + ), + ], +) +def test_component_dependency_required_for_catalog( + config: Config, requiredif: list[str], required: bool +): + d = _make_dep( + "argocd", + "https://github.com/projectsyn/component-argocd.git", + requiredif=requiredif, + ) + + facts = { + "distribution": "talos", + } + cparams = { + "foo": "bar", + "param": "value", + } + + assert d.required_for_catalog(config, cparams, facts) == required + + +@pytest.mark.parametrize( + "requiredif,expected_error", + [ + ( + ["facts.distribution"], + "Component dependency `requiredif` CEL expression must evaluate to a boolean", + ), + ( + ["facts.cloud == 'cloudscale'"], + 'Evaluation failed for `requiredif` CEL expression: NOT_FOUND: Key not found in map : "cloud"\'', + ), + ], +) +def test_component_dependency_required_for_catalog_errors( + config: Config, requiredif: list[str], expected_error: str +): + d = _make_dep( + "argocd", + "https://github.com/projectsyn/component-argocd.git", + requiredif=requiredif, + ) + + facts = { + "distribution": "talos", + } + cparams = { + "foo": "bar", + "param": "value", + } + + with pytest.raises(ValueError) as e: + d.required_for_catalog(config, cparams, facts) + + assert expected_error in str(e) + + +@pytest.mark.parametrize( + "dep,expected", + [ + ( + _make_dep( + "argocd", + "https://github.com/projectsyn/component-argocd.git", + ), + { + "url": "https://github.com/projectsyn/component-argocd.git", + "version": "master", + }, + ), + ( + _make_dep( + "argocd", + "https://github.com/projectsyn/component-argocd.git", + minverspec="v1.0.0", + ), + { + "url": "https://github.com/projectsyn/component-argocd.git", + "version": "v1.0.0", + }, + ), + ( + _make_dep( + "argocd", + "https://github.com/projectsyn/component-argocd.git", + path="foobar", + ), + { + "url": "https://github.com/projectsyn/component-argocd.git", + "version": "master", + "path": "foobar", + }, + ), + ], +) +def test_component_dependency_component_entry(dep, expected): + assert dep.component_entry == expected + + +def _make_inv(config: Config, tmp_path: Path): + cluster = { + "parameters": { + "components": { + "test-component-1": { + "url": "https://github.com/projectsyn/component-tc1.git", + "version": "v1.0.1", + }, + "test-component-2": { + "url": "https://github.com/projectsyn/component-tc2.git", + "version": "v1.0.2", + }, + "test-component-3": { + "url": "https://github.com/projectsyn/component-tc3.git", + "version": "v1.0.3", + }, + "test-component-4": { + "url": "https://github.com/projectsyn/component-tc4.git", + "version": "v1.0.4", + }, + "test-component-5": { + "url": "https://github.com/projectsyn/component-tc5.git", + "version": "v1.0.5", + }, + }, + "facts": { + "distribution": "talos", + "cloud": "cloudscale", + }, + "test_component_1": {}, + "test_component_2": {}, + "test_component_3": { + "tc4_enabled": True, + }, + "test_component_4": {}, + "test_component_5": {}, + } + } + + tc1 = copy.deepcopy(cluster) + tc1["parameters"]["commodore"] = { + "dependencies": { + "test-component-2": { + "url": "https://github.com/projectsyn/component-tc2.git", + }, + "test-component-3": { + "url": "https://github.com/projectsyn/component-tc2.git", + "minversion": "v1.1.0", + "requiredif": "facts.distribution == 'openshift4'", + }, + } + } + tc2 = copy.deepcopy(cluster) + tc3 = copy.deepcopy(cluster) + tc3["parameters"]["commodore"] = { + "dependencies": { + "test-component-4": { + "url": "https://github.com/projectsyn/component-tc4.git", + "requiredif": "facts.distribution == 'talos' && config.tc4_enabled", + }, + } + } + tc4 = copy.deepcopy(cluster) + tc4["parameters"]["commodore"] = { + "dependencies": { + "test-component-5": { + "url": "https://github.com/projectsyn/component-tc5.git", + "minversion": "v1.0.1", + } + } + } + tc5 = copy.deepcopy(cluster) + + inv = { + "cluster": cluster, + "test-component-1": tc1, + "test-component-2": tc2, + "test-component-3": tc3, + "test-component-4": tc4, + "test-component-5": tc5, + } + for i in range(1, 6): + cdep = MultiDependency( + f"https://github.com/projectsyn/component-tc{i}.git", + tmp_path / "dependencies", + ) + c = Component(f"test-component-{i}", dependency=cdep, work_dir=tmp_path) + config.register_component(c) + config.register_component_aliases( + { + "test-component-1": "test-component-1", + "test-component-2": "test-component-2", + "test-component-3": "test-component-3", + "test-component-4": "test-component-4", + "test-component-5": "test-component-5", + } + ) + return inv + + +def test_collect_catalog_dependencies(tmp_path: Path, config: Config): + inv = _make_inv(config, tmp_path) + + deps = component_dependency.collect_catalog_dependencies(config, inv) + + expected_deps = { + "test-component-2": _make_dep( + "test-component-2", + "https://github.com/projectsyn/component-tc2.git", + instances=["test-component-1"], + ), + "test-component-4": _make_dep( + "test-component-4", + "https://github.com/projectsyn/component-tc4.git", + instances=["test-component-3"], + requiredif=["facts.distribution == 'talos' && config.tc4_enabled"], + ), + "test-component-5": _make_dep( + "test-component-5", + "https://github.com/projectsyn/component-tc5.git", + instances=["test-component-4"], + minverspec="v1.0.1", + ), + } + + assert deps == expected_deps + + +def test_validate_catalog_dependencies(config: Config, tmp_path: Path): + inv = _make_inv(config, tmp_path) + component_dependency.validate_catalog_dependencies(config, inv)