Implemented spawn config validation

Signed-off-by: Victor Moene <victor.moene@northern.tech>

Author: victormlg

pyproject.toml

@@ -15,6 +15,8 @@ dependencies = [
     "tree-sitter-cfengine>=1.1.8",
     "tree-sitter>=0.25",
     "markdown-it-py>=3.0.0",
+    "pyyaml>=6.0.3",
+    "pydantic>=2.12.5",
 ]
 classifiers = [
   "Development Status :: 3 - Alpha",

src/cfengine_cli/commands.py

@@ -2,6 +2,7 @@
 import os
 import re
 import json
+import yaml
 from cfengine_cli.profile import profile_cfengine, generate_callstack
 from cfengine_cli.dev import dispatch_dev_subcommand
 from cfengine_cli.lint import lint_folder, lint_single_arg
@@ -14,6 +15,7 @@
     format_policy_fin_fout,
 )
 from cfengine_cli.utils import UserError
+from cfengine_cli.up import validate_config
 from cfbs.utils import find
 from cfbs.commands import build_command
 from cf_remote.commands import deploy as deploy_command
@@ -148,3 +150,18 @@ def profile(args) -> int:
         generate_callstack(data, args.flamegraph)
 
     return 0
+
+
+def up(config) -> int:
+    content = None
+    try:
+        with open(config, "r") as f:
+            content = yaml.safe_load(f)
+    except yaml.YAMLError:
+        raise UserError("'%s' is not a valid yaml config" % config)
+    except FileNotFoundError:
+        raise UserError("'%s' doesn't exist" % config)
+
+    print("Starting VMs...")
+    validate_config(content)
+    return 0

src/cfengine_cli/main.py

@@ -114,6 +114,12 @@ def _get_arg_parser():
         dest="minimum_version",
     )
 
+    up_parser = subp.add_parser(
+        "up", help="Spawn and install with cf-remote from a yaml config"
+    )
+    up_parser.add_argument(
+        "config", default="config.yaml", nargs="?", help="Path to yaml config"
+    )
     return ap
 
 
@@ -147,6 +153,8 @@ def run_command_with_args(args) -> int:
         return commands.dev(args.dev_command, args)
     if args.command == "profile":
         return commands.profile(args)
+    if args.command == "up":
+        return commands.up(args.config)
     raise UserError(f"Unknown command: '{args.command}'")
 
 

src/cfengine_cli/up.py

@@ -0,0 +1,225 @@
+from pydantic import BaseModel, model_validator, ValidationError, Field
+from typing import Union, Literal, Optional, List, Annotated
+from functools import reduce
+from cf_remote import log
+
+import cfengine_cli.validate as validate
+from cfengine_cli.utils import UserError
+
+
+# Forces pydantic to throw validation error if config contains unknown keys
+class NoExtra(BaseModel, extra="forbid"):
+    pass
+
+
+class Config(NoExtra):
+    pass
+
+
+class AWSConfig(Config):
+    image: str
+    size: Literal["micro", "xlarge"] = "micro"
+
+    @model_validator(mode="after")
+    def check_aws_config(self):
+        validate.validate_aws_image(self.image)
+        return self
+
+
+class VagrantConfig(Config):
+    box: str
+    memory: int = 512
+    cpus: int = 1
+    sync_folder: Optional[str] = None
+    provision: Optional[str] = None
+
+    @model_validator(mode="after")
+    def check_vagrant_config(self):
+        if self.memory < 512:
+            raise UserError("Cannot allocate less than 512MB to a Vagrant VM")
+        if self.cpus < 1:
+            raise UserError("Cannot use less than 1 cpu per Vagrant VM")
+
+        validate.validate_vagrant_box(self.box)
+
+        return self
+
+
+class GCPConfig(Config):
+    image: str  # There is no list of avalaible GCP platforms to validate against yet
+    network: Optional[str] = None
+    public_ip: bool = True
+    size: str = "n1-standard-1"
+
+
+class AWSProvider(Config):
+    provider: Literal["aws"]
+    aws: AWSConfig
+
+    @model_validator(mode="after")
+    def check_aws_provider(self):
+        validate.validate_aws_credentials()
+        return self
+
+
+class GCPProvider(Config):
+    provider: Literal["gcp"]
+    gcp: GCPConfig
+
+    @model_validator(mode="after")
+    def check_gcp_provider(self):
+        validate.validate_gcp_credentials()
+        return self
+
+
+class VagrantProvider(Config):
+    provider: Literal["vagrant"]
+    vagrant: VagrantConfig
+
+
+class SaveMode(Config):
+    mode: Literal["save"]
+    hosts: List[str]
+
+
+class SpawnMode(Config):
+    mode: Literal["spawn"]
+    # "Field" forces pydantic to report errors on the branch defined by the field "provider"
+    spawn: Annotated[
+        Union[VagrantProvider, AWSProvider, GCPProvider],
+        Field(discriminator="provider"),
+    ]
+    count: int
+
+    @model_validator(mode="after")
+    def check_spawn_config(self):
+        if self.count < 1:
+            raise UserError("Cannot spawn less than 1 instance")
+        return self
+
+
+class CFEngineConfig(Config):
+    version: Optional[str] = None
+    bootstrap: Optional[str] = None
+    edition: Literal["community", "enterprise"] = "enterprise"
+    remote_download: bool = False
+    hub_package: Optional[str] = None
+    client_package: Optional[str] = None
+    package: Optional[str] = None
+    demo: bool = False
+
+    @model_validator(mode="after")
+    def check_cfengine_config(self):
+        packages = [self.package, self.hub_package, self.client_package]
+        for p in packages:
+            validate.validate_package(p, self.remote_download)
+
+        if self.version and any(packages):
+            log.warning("Specifying package overrides cfengine version")
+
+        validate.validate_version(self.version, self.edition)
+        validate.validate_state_bootstrap(self.bootstrap)
+
+        return self
+
+
+class GroupConfig(Config):
+    role: Literal["client", "hub"]
+    # "Field" forces pydantic to report errors on the branch defined by the field "provider"
+    source: Annotated[Union[SaveMode, SpawnMode], Field(discriminator="mode")]
+    cfengine: Optional[CFEngineConfig] = None
+    scripts: Optional[List[str]] = None
+
+    @model_validator(mode="after")
+    def check_group_config(self):
+        if (
+            self.role == "hub"
+            and self.source.mode == "spawn"
+            and self.source.count != 1
+        ):
+            raise UserError("A hub can only have one host")
+
+        return self
+
+
+def rgetattr(obj, attr, *args):
+    def _getattr(obj, attr):
+        return getattr(obj, attr, *args)
+
+    return reduce(_getattr, [obj] + attr.split("."))
+
+
+class Group:
+    """
+    All group-specific data:
+    - Vagrantfile
+    Config that declares it:
+    - provider, count, cfengine version, role, ...
+    """
+
+    def __init__(self, config: GroupConfig):
+        self.config = config
+        self.hosts = []
+
+
+class Host:
+    """
+    All host-specific data:
+    - user, ip, ssh config, OS, uuid, ...
+    """
+
+    def __init__(self):
+        pass
+
+
+def _resolve_templates(parent, templates):
+    if not parent:
+        return
+    if isinstance(parent, dict):
+        for key, value in parent.items():
+            if isinstance(value, str) and value in templates:
+                parent[key] = templates[value]
+            else:
+                _resolve_templates(value, templates)
+    if isinstance(parent, list):
+        for value in parent:
+            _resolve_templates(value, templates)
+
+
+def validate_config(content):
+    if not content:
+        raise UserError("Empty spawn config")
+
+    if "groups" not in content:
+        raise UserError("Missing 'groups' key in spawn config")
+
+    groups = content["groups"]
+    templates = content.get("templates")
+    if templates:
+        _resolve_templates(groups, templates)
+
+    if not isinstance(groups, list):
+        groups = [groups]
+
+    state = {}
+    try:
+        for g in groups:
+            if len(g) != 1:
+                raise UserError(
+                    "Too many keys in group definition: {}".format(
+                        ", ".join(list(g.keys()))
+                    )
+                )
+
+            for k, v in g.items():
+                state[k] = Group(GroupConfig(**v))
+
+    except ValidationError as v:
+        msgs = []
+        for err in v.errors():
+            msgs.append(
+                "{}. Input '{}' at location '{}'".format(
+                    err["msg"], err["input"], err["loc"]
+                )
+            )
+        raise UserError("\n".join(msgs))