Implement support for Commodore component dependencies (SDD #0034)

commodore/cluster.py

@@ -5,7 +5,7 @@
 import textwrap
 
 from datetime import datetime
-from typing import Any, Optional, Union
+from typing import Any, Iterable, Optional, Union
 
 import click
 
@@ -200,6 +200,7 @@ def render_target(
     target: str,
     components: dict[str, Component],
     component: Optional[str] = None,
+    extra_classes: Optional[Iterable[str]] = None,
 ):
     if not component:
         component = target
@@ -221,6 +222,8 @@ def render_target(
             click.secho(f" > Default file for class {c} missing", fg="yellow")
 
     classes.append("global.commodore")
+    if extra_classes:
+        classes.extend(extra_classes)
 
     if not bootstrap:
         if not inv.component_file(target).is_file():
@@ -232,12 +235,21 @@ def render_target(
     return generate_target(inv, target, components, classes, component)
 
 
-def update_target(cfg: Config, target: str, component: Optional[str] = None):
+def update_target(
+    cfg: Config,
+    target: str,
+    component: Optional[str] = None,
+    extra_classes: Optional[Iterable[str]] = None,
+):
     click.secho(f"Updating Kapitan target for {target}...", bold=True)
     file = cfg.inventory.target_file(target)
     os.makedirs(file.parent, exist_ok=True)
     targetdata = render_target(
-        cfg.inventory, target, cfg.get_components(), component=component
+        cfg.inventory,
+        target,
+        cfg.get_components(),
+        component=component,
+        extra_classes=extra_classes,
     )
     yaml_dump(targetdata, file)
 

commodore/compile.py

@@ -25,6 +25,7 @@
     verify_version_overrides,
 )
 from .dependency_mgmt.component_library import create_component_library_aliases
+from .dependency_mgmt.component_dependency import validate_catalog_dependencies
 from .dependency_mgmt.jsonnet_bundler import (
     fetch_jsonnet_libraries,
     jsonnet_dependencies,
@@ -241,6 +242,9 @@ def setup_compile_environment(config: Config) -> tuple[dict[str, Any], Iterable[
     # Raise exception if component version override without URL is present in the
     # hierarchy.
     verify_version_overrides(cluster_parameters, config.get_component_aliases())
+    # Raise exception if the catalog violates any component dependency version
+    # requirements.
+    validate_catalog_dependencies(config, inventory)
 
     for component in config.get_components().values():
         ckey = component.parameters_key

commodore/component/compile.py

@@ -6,19 +6,27 @@
 from collections.abc import Iterable
 from pathlib import Path as P
 from textwrap import dedent
-from typing import Optional
+from typing import Any, Optional
 
 import click
 import git
 
-from commodore.cluster import generate_target
+from commodore.cluster import update_target
 from commodore.config import Config
 from commodore.component import Component
+from commodore.dependency_mgmt import fetch_components, create_component_symlinks
+from commodore.dependency_mgmt.component_dependency import (
+    collect_catalog_dependencies,
+    ComponentDependency,
+)
 from commodore.dependency_mgmt.component_library import (
     validate_component_library_name,
     create_component_library_aliases,
 )
-from commodore.dependency_mgmt.jsonnet_bundler import fetch_jsonnet_libraries
+from commodore.dependency_mgmt.jsonnet_bundler import (
+    fetch_jsonnet_libraries,
+    jsonnet_dependencies,
+)
 from commodore.helpers import kapitan_inventory, kapitan_compile, relsymlink, yaml_dump
 from commodore.inventory import Inventory
 from commodore.inventory.lint import check_removed_reclass_variables
@@ -40,7 +48,6 @@ def compile_component(
     component_path = P(component_path_).resolve()
     value_files = [P(f).resolve() for f in value_files_]
     search_paths = [P(d).resolve() for d in search_paths_]
-    search_paths.append(component_path / "vendor")
     output_path = P(output_path_).resolve()
 
     if not component_name:
@@ -59,20 +66,24 @@ def compile_component(
         )
 
     temp_dir = P(tempfile.mkdtemp(prefix="component-")).resolve()
+    search_paths.append(temp_dir / "vendor")
     config.work_dir = temp_dir
     try:
         if config.debug:
             click.echo(f"   > Created temp workspace: {config.work_dir}")
         inv = config.inventory
         inv.ensure_dirs()
+        inv.global_config_dir.mkdir()
+        yaml_dump({}, inv.global_config_dir / "commodore.yml")
         search_paths.append(inv.dependencies_dir)
         component = _setup_component(
             config,
             component_name,
             instance_name,
             component_path,
         )
-        _prepare_kapitan_inventory(inv, component, value_files, instance_name)
+        config.register_component(component)
+        _prepare_kapitan_inventory(config, component, value_files, instance_name)
 
         # Raise error if component uses removed reclass parameters
         check_removed_reclass_variables(
@@ -81,21 +92,50 @@ def compile_component(
             [component.defaults_file, component.class_file] + value_files,
         )
 
-        # Verify component alias
+        # Fetch and install component dependencies
+        click.secho(
+            f"Discovering component dependencies for {instance_name}...", bold=True
+        )
         nodes = kapitan_inventory(config)
+        component_deps = collect_catalog_dependencies(config, nodes)
+        if len(component_deps) > 0:
+            component_deps["argocd"] = ComponentDependency.parse(
+                component.name,
+                "argocd",
+                {"url": "https://github.com/projectsyn/component-argocd.git"},
+            )
+
+            _setup_dependencies(inv, component_deps)
+            update_target(config, inv.bootstrap_target)
+
+            fetch_components(config)
+
+            update_target(config, inv.bootstrap_target)
+            _prepare_kapitan_inventory(config, component, value_files, instance_name)
+
+            cluster_parameters = kapitan_inventory(config)[inv.bootstrap_target][
+                "parameters"
+            ]
+            nodes = kapitan_inventory(config)
+        else:
+            _setup_fake_argocd_lib(inv)
+            cluster_parameters = kapitan_inventory(config)[instance_name]["parameters"]
+            create_component_symlinks(config, component)
+            search_paths.append(component_path / "vendor")
+
+        # Fetch Jsonnet dependencies
+        for component in config.get_components().values():
+            ckey = component.parameters_key
+            component.render_jsonnetfile_json(cluster_parameters[ckey])
+
+        fetch_jsonnet_libraries(config.work_dir, deps=jsonnet_dependencies(config))
+
+        # Verify component alias
         config.verify_component_aliases(nodes, bootstrap_target=instance_name)
 
         cluster_params = nodes[instance_name]["parameters"]
         create_component_library_aliases(config, cluster_params)
 
-        # Render jsonnetfile.jsonnet if necessary
-        component_params = nodes[instance_name]["parameters"].get(
-            component_name.replace("-", "_"), {}
-        )
-        component.render_jsonnetfile_json(component_params)
-        # Fetch Jsonnet libs
-        fetch_jsonnet_libraries(component_path)
-
         # Compile component
         kapitan_compile(
             config,
@@ -180,14 +220,20 @@ def _setup_component(
 
 
 def _prepare_kapitan_inventory(
-    inv: Inventory, component: Component, value_files: Iterable[P], instance_name: str
+    config: Config,
+    component: Component,
+    value_files: Iterable[P],
+    instance_name: str,
 ):
     """
     Setup Kapitan inventory.
 
     Create component symlinks, values file symlinks, setup params class with fake values
     and Kapitan target for the component, create a fake `lib/argocd.libjsonnet`.
     """
+
+    inv = config.inventory
+
     component_class_file = component.class_file
     component_defaults_file = component.defaults_file
     if not component_class_file.exists():
@@ -200,12 +246,14 @@ def _prepare_kapitan_inventory(
         )
 
     # Create class symlink
-    relsymlink(component_class_file, inv.components_dir)
+    relsymlink(
+        component_class_file, inv.components_dir, dest_name=f"{instance_name}.yml"
+    )
     # Create defaults symlink
     relsymlink(
         component_defaults_file,
         inv.defaults_dir,
-        dest_name=f"{component.name}.yml",
+        dest_name=f"{instance_name}.yml",
     )
     # Create component symlink
     relsymlink(component.target_directory, inv.dependencies_dir, component.name)
@@ -229,9 +277,6 @@ def _prepare_kapitan_inventory(
                     "cloud": "cloudscale",
                     "region": "rma1",
                 },
-                "argocd": {
-                    "namespace": "test",
-                },
                 "components": {
                     component.name: {
                         "url": f"https://example.com/{component.name}.git",
@@ -251,18 +296,10 @@ def _prepare_kapitan_inventory(
 
     # Create test target
     value_classes = [f"{c.stem}" for c in value_files]
-    classes = [
-        f"params.{inv.bootstrap_target}",
-        f"defaults.{component.name}",
-        f"components.{component.name}",
-    ] + value_classes
-    yaml_dump(
-        generate_target(
-            inv, instance_name, {component.name: component}, classes, component.name
-        ),
-        inv.target_file(instance_name),
-    )
+    update_target(config, instance_name, component.name, value_classes)
+
 
+def _setup_fake_argocd_lib(inv: Inventory):
     # Fake Argo CD lib
     # We plug "fake" Argo CD library here because every component relies on it
     # and we don't want to provide it every time when compiling a single component.
@@ -275,3 +312,13 @@ def _prepare_kapitan_inventory(
               App: ArgoApp,
               Project: ArgoProject,
             }"""))
+
+
+def _setup_dependencies(inv: Inventory, dependencies: dict[str, ComponentDependency]):
+    dependencies_yaml: dict[str, Any] = {
+        "applications": list(dependencies.keys()),
+        "parameters": {
+            "components": {dn: dep.component_entry for dn, dep in dependencies.items()}
+        },
+    }
+    yaml_dump(dependencies_yaml, inv.global_config_dir / "commodore.yml")

commodore/config.py

@@ -344,7 +344,7 @@ def inventory(self):
     def update_verbosity(self, verbose):
         self._verbose += verbose
 
-    def get_components(self):
+    def get_components(self) -> dict[str, Component]:
         return self._components
 
     def register_component(self, component: Component):
@@ -396,7 +396,7 @@ def register_dependency_repo(self, repo_url: str) -> MultiDependency:
             dep.url = repo_url
         return dep
 
-    def get_component_aliases(self):
+    def get_component_aliases(self) -> dict[str, str]:
         return self._component_aliases
 
     def register_component_aliases(self, aliases: dict[str, str]):

commodore/dependency_mgmt/__init__.py

@@ -2,7 +2,7 @@
 
 import itertools
 from concurrent.futures import ThreadPoolExecutor
-from typing import Callable, Iterable
+from typing import Callable, Iterable, Optional
 
 import click
 from click import ClickException
@@ -77,7 +77,7 @@ def create_package_symlink(cfg, pname: str, package: Package):
     relsymlink(package.target_dir, cfg.inventory.classes_dir, dest_name=pname)
 
 
-def fetch_components(cfg: Config):
+def fetch_components(cfg: Config, bootstrap_target: Optional[str] = None):
     """
     Download all components required by target.
 
@@ -90,7 +90,7 @@ def fetch_components(cfg: Config):
     component_names, component_aliases = _discover_components(cfg)
     click.secho("Registering component aliases...", bold=True)
     cfg.register_component_aliases(component_aliases)
-    cspecs = _read_components(cfg, component_aliases)
+    cspecs = _read_components(cfg, component_aliases, bootstrap_target)
     click.secho("Fetching components...", bold=True)
 
     deps: dict[str, list] = {}
@@ -184,7 +184,7 @@ def do_parallel(fun: Callable[[Config, Iterable], None], cfg: Config, data: Iter
         list(exe.map(fun, itertools.repeat(cfg), data))
 
 
-def register_components(cfg: Config):
+def register_components(cfg: Config, bootstrap_target: Optional[str] = None):
     """
     Discover components in the inventory, and register them if the
     corresponding directory in `dependencies/` exists.
@@ -194,7 +194,7 @@ def register_components(cfg: Config):
     click.secho("Discovering included components...", bold=True)
     try:
         components, component_aliases = _discover_components(cfg)
-        cspecs = _read_components(cfg, component_aliases)
+        cspecs = _read_components(cfg, component_aliases, bootstrap_target)
     except KeyError as e:
         raise click.ClickException(f"While discovering components: {e}")
     click.secho("Registering components and aliases...", bold=True)