diff --git a/.github/workflows/ansible-lint.yml b/.github/workflows/ansible-lint.yml index c3cceaf..f62643b 100644 --- a/.github/workflows/ansible-lint.yml +++ b/.github/workflows/ansible-lint.yml @@ -14,7 +14,21 @@ jobs: with: persist-credentials: false - - name: Lint Ansible Playbook - uses: ansible/ansible-lint@5fac056c45595896c973fbde871f01f6cb14d74c + # The ansible-lint GitHub action installs via uv with a lock that tracks the newest + # ansible-core; ansible-galaxy collection install then hits Galaxy KeyError: 'results' + # on some 2.19+/2.20 clients. Pin ansible-core to the 2.18 series for a stable Galaxy client. + - name: Set up Python + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: - setup_python: "true" + python-version: "3.12" + + - name: Install ansible-lint and Galaxy-compatible ansible-core + run: | + python -m pip install --upgrade pip + pip install ansible-lint "ansible-core>=2.18.0,<2.19.0" + + - name: Install Ansible Galaxy collection dependencies + run: ansible-galaxy collection install -r requirements.yml + + - name: Run ansible-lint + run: ansible-lint roles playbooks plugins diff --git a/.github/workflows/jsonschema.yaml b/.github/workflows/jsonschema.yaml index b9c1424..80a3433 100644 --- a/.github/workflows/jsonschema.yaml +++ b/.github/workflows/jsonschema.yaml @@ -32,4 +32,4 @@ jobs: - name: Verify secrets json schema run: | set -e - for i in values-secret-v2-base values-secret-v2-generic-onlygenerate values-secret-v2-block-yamlstring; do echo "$i"; check-jsonschema --fill-defaults --schemafile ./roles/vault_utils/values-secrets.v2.schema.json "tests/unit/v2/$i.yaml"; done + for i in values-secret-v2-base values-secret-v2-generic-onlygenerate values-secret-v2-block-yamlstring values-secret-v2-bootstrap-and-late values-secret-v2-bootstrap-only; do echo "$i"; check-jsonschema --fill-defaults --schemafile ./roles/vault_utils/values-secrets.v2.schema.json "tests/unit/v2/$i.yaml"; done diff --git a/.gitignore b/.gitignore index e515a54..5231ca4 100644 --- a/.gitignore +++ b/.gitignore @@ -13,3 +13,4 @@ super-linter.log golang-external-secrets/Chart.lock hashicorp-vault/Chart.lock tests/output +ansible_collections/ diff --git a/Makefile b/Makefile index f62ebfb..fff9006 100644 --- a/Makefile +++ b/Makefile @@ -48,4 +48,4 @@ test: ansible-sanitytest ansible-unittest .PHONY: check-jsonschema check-jsonschema: ## Runs check-jsonschema against all unit test files except known broken ones set -e; \ - for i in values-secret-v2-base values-secret-v2-generic-onlygenerate values-secret-v2-block-yamlstring; do echo "$$i"; check-jsonschema --schemafile ./roles/vault_utils/values-secrets.v2.schema.json "tests/unit/v2/$$i.yaml"; done + for i in values-secret-v2-base values-secret-v2-generic-onlygenerate values-secret-v2-block-yamlstring values-secret-v2-bootstrap-and-late values-secret-v2-bootstrap-only; do echo "$$i"; check-jsonschema --schemafile ./roles/vault_utils/values-secrets.v2.schema.json "tests/unit/v2/$$i.yaml"; done diff --git a/README.md b/README.md index 68af1ea..be97726 100644 --- a/README.md +++ b/README.md @@ -8,3 +8,14 @@ The main purpose of this collections are to: loading local secrets files into VP secrets stores. 2. Help manage imperative and other utility functions of the cluster + +## Pattern repository directory (`pattern_dir`) + +Playbooks need the path to your pattern Git checkout (where **`values-global.yaml`** +and related files live). Resolution order: extra var **`pattern_dir`**, environment +variable **`PATTERN_DIR`**, then **`PWD`** and **`pwd`**. + +When running from the imperative container or another fixed working directory, +pass the repository root explicitly, for example **`-e pattern_dir=/git/repo`** (or add +equivalent extra vars via **`clusterGroup.imperative.extraPlaybookArgs`** in the +clustergroup chart). diff --git a/playbooks/determine_pattern_dir.yml b/playbooks/determine_pattern_dir.yml index 17b2cd7..b6ef4c2 100644 --- a/playbooks/determine_pattern_dir.yml +++ b/playbooks/determine_pattern_dir.yml @@ -1,17 +1,20 @@ --- +# Same resolution as roles/pattern_settings/tasks/resolve_overrides.yml and README.md: +# extra var pattern_dir, then env PATTERN_DIR, then env PWD, then output of pwd (trimmed). - name: Determine pattern dir hosts: localhost connection: local gather_facts: false become: false - vars: - pattern_dir: '' tasks: - - name: Fail if directory is not set - ansible.builtin.fail: - msg: "pattern_dir variable must be set" - when: pattern_dir | length == 0 + - name: Resolve pattern_dir (same as pattern_settings) + ansible.builtin.include_role: + name: pattern_settings + tasks_from: resolve_overrides - - name: Set pattern_dir fact for future plays - ansible.builtin.set_fact: - pattern_dir: '{{ pattern_dir }}' + - name: Fail if pattern_dir is still empty after resolution + ansible.builtin.fail: + msg: >- + Could not resolve pattern_dir. Set -e pattern_dir=/path/to/pattern, export PATTERN_DIR, + or run from your pattern repository so PWD / pwd points at the directory that contains values-global.yaml. + when: pattern_dir | default('', true) | string | trim | length == 0 diff --git a/playbooks/determine_secretstore_backend.yml b/playbooks/determine_secretstore_backend.yml index f1c0410..43d4ae9 100644 --- a/playbooks/determine_secretstore_backend.yml +++ b/playbooks/determine_secretstore_backend.yml @@ -7,8 +7,6 @@ connection: local gather_facts: false become: false - vars: - pattern_dir: "." tasks: - name: Set fact for secretStore backend ansible.builtin.set_fact: diff --git a/playbooks/display_secrets_info.yml b/playbooks/display_secrets_info.yml index 8ce8619..a84f3fd 100644 --- a/playbooks/display_secrets_info.yml +++ b/playbooks/display_secrets_info.yml @@ -23,19 +23,54 @@ ansible.builtin.include_role: name: find_vp_secrets + # find_vp_secrets may set values_secrets_data as a dict (from_yaml) or as raw text; parse_secrets_info needs a YAML string. # find_vp_secrets will return a plaintext data structure called values_secrets_data # This will allow us to determine schema version and which backend to use - name: Determine how to load secrets ansible.builtin.set_fact: secrets_yaml: "{{ values_secrets_data if values_secrets_data is not string else values_secrets_data | from_yaml }}" - - name: Parse secrets data + - name: Normalize values-secret content for parse_secrets_info + ansible.builtin.set_fact: + values_secrets_plaintext_for_parse: >- + {{ values_secrets_data if values_secrets_data is string else (values_secrets_data | to_yaml) }} + + - name: Resolve secrets backing store for parse module + ansible.builtin.set_fact: + secrets_backing_store_for_parse: >- + {{ secrets_backing_store + | default(values_global.global.secretStore.backend | default('vault'), true) }} + + - name: Parse early-phase (bootstrap) secrets no_log: '{{ hide_sensitive_output }}' parse_secrets_info: - values_secrets_plaintext: "{{ values_secrets_data }}" - secrets_backing_store: "{{ secrets_backing_store }}" - register: secrets_results + values_secrets_plaintext: "{{ values_secrets_plaintext_for_parse }}" + secrets_backing_store: "{{ secrets_backing_store_for_parse }}" + secrets_phase: early + register: secrets_results_early + + - name: Parse late-phase secrets + no_log: '{{ hide_sensitive_output }}' + parse_secrets_info: + values_secrets_plaintext: "{{ values_secrets_plaintext_for_parse }}" + secrets_backing_store: "{{ secrets_backing_store_for_parse }}" + secrets_phase: late + register: secrets_results_late + + - name: Assemble secrets display (early and late phases) + ansible.builtin.set_fact: + secrets_display_by_phase: + vault_policies: "{{ secrets_results_late.vault_policies | default(secrets_results_early.vault_policies | default({})) }}" + secret_store_namespace: "{{ secrets_results_late.secret_store_namespace | default('validated-patterns-secrets') }}" + early_phase: + parsed_secrets: "{{ secrets_results_early.parsed_secrets | default({}) }}" + kubernetes_secret_objects: "{{ secrets_results_early.kubernetes_secret_objects | default([]) }}" + unique_vault_prefixes: "{{ secrets_results_early.unique_vault_prefixes | default([]) }}" + late_phase: + parsed_secrets: "{{ secrets_results_late.parsed_secrets | default({}) }}" + kubernetes_secret_objects: "{{ secrets_results_late.kubernetes_secret_objects | default([]) }}" + unique_vault_prefixes: "{{ secrets_results_late.unique_vault_prefixes | default([]) }}" - - name: Display secrets data + - name: Display secrets data (early and late phases) ansible.builtin.debug: - var: secrets_results + var: secrets_display_by_phase diff --git a/playbooks/install.yml b/playbooks/install.yml index 2ff805a..4442a48 100644 --- a/playbooks/install.yml +++ b/playbooks/install.yml @@ -1,8 +1,11 @@ --- +- name: Load bootstrap secrets early phase + ansible.builtin.import_playbook: load_bootstrap_secrets.yml + - name: Install the pattern via pattern-install chart ansible.builtin.import_playbook: operator_deploy.yml -- name: Load secrets (if not explicitly disabled in values-global.yaml) +- name: Load secrets late phase (if not explicitly disabled in values-global.yaml) ansible.builtin.import_playbook: load_secrets.yml - name: Wait for pattern to finish installation (all Argo apps should be healthy/synced) diff --git a/playbooks/load_bootstrap_secrets.yml b/playbooks/load_bootstrap_secrets.yml new file mode 100644 index 0000000..3da9c4c --- /dev/null +++ b/playbooks/load_bootstrap_secrets.yml @@ -0,0 +1,19 @@ +--- +# Loads only bootstrap_secrets (early phase), before pattern operator install. +# Always uses the Kubernetes / none injector (never vault_utils), even when values-global uses vault. +# Vault is not available yet at this stage; bootstrap material is never written to Vault from here. +# Runs even when .global.secretLoader.disabled is true (late-phase loading still honors that flag). +# Use playbooks/load_secrets.yml for late-phase secrets. +- name: Decide whether to load bootstrap secrets + hosts: localhost + connection: local + gather_facts: false + roles: + - role: pattern_settings + + tasks: + - name: Load bootstrap secrets early phase + ansible.builtin.include_role: + name: load_secrets + vars: + secrets_phase: early diff --git a/playbooks/load_secrets.yml b/playbooks/load_secrets.yml index e8295b3..c5db04a 100644 --- a/playbooks/load_secrets.yml +++ b/playbooks/load_secrets.yml @@ -1,5 +1,9 @@ --- -- name: Decide whether to load secrets +# Loads late-phase `secrets` entries. If `bootstrap_secrets` were not applied earlier in +# the same playbook on this host (see `bootstrap_secrets_loaded_this_run`), the +# load_secrets role applies them once before late parse/inject. Use load_bootstrap_secrets.yml +# for explicit early-only bootstrap loading (e.g. before operator install). +- name: Decide whether to load late-phase secrets hosts: localhost connection: local gather_facts: false @@ -11,13 +15,14 @@ ansible.builtin.set_fact: secret_loader_disabled: "{{ values_global.global.secretLoader.disabled | default(false) | bool }}" - - name: Load secrets (when enabled) + - name: Load late-phase secrets (when enabled) ansible.builtin.include_role: name: load_secrets when: not secret_loader_disabled - - name: Print secret loading disabled message + - name: Print late-phase secret loading disabled message ansible.builtin.debug: msg: | - Secrets loading is currently disabled. To enable, update the value of '.global.secretLoader.disabled' in 'values-global.yaml' to 'false'. + Late-phase secrets loading is currently disabled. Bootstrap secrets were still applied when present. + To enable late-phase loading, update the value of '.global.secretLoader.disabled' in 'values-global.yaml' to 'false'. when: secret_loader_disabled diff --git a/playbooks/process_secrets.yml b/playbooks/process_secrets.yml index 6329dda..ca1c318 100644 --- a/playbooks/process_secrets.yml +++ b/playbooks/process_secrets.yml @@ -9,6 +9,7 @@ pattern_dir: '.' secrets_backing_store: 'vault' tasks_from: 'push_parsed_secrets' + secrets_phase: late tasks: - name: "Run secret-loading pre-requisites" ansible.builtin.include_role: @@ -23,21 +24,49 @@ ansible.builtin.set_fact: secrets_yaml: "{{ values_secrets_data if values_secrets_data is not string else values_secrets_data | from_yaml }}" + - name: Ensure bootstrap secrets before late phase when not already applied this run + ansible.builtin.include_role: + name: load_secrets + tasks_from: ensure_bootstrap_before_late.yml + when: + - secrets_phase | default('late') == 'late' + - not (bootstrap_secrets_loaded_this_run | default(false)) + - (secrets_yaml.version | default('2.0')) is version('2.0', '>=') + - (secrets_yaml.bootstrap_secrets | default([]) | length) > 0 + - name: Parse secrets data no_log: '{{ hide_sensitive_output | default(true) }}' parse_secrets_info: values_secrets_plaintext: "{{ values_secrets_data }}" secrets_backing_store: "{{ secrets_backing_store }}" + secrets_phase: "{{ secrets_phase }}" register: secrets_results - # Use the k8s secrets loader when explicitly requested - - name: Determine role to use to load secrets + - name: Assert values-secret v2 when loading bootstrap (early) secrets + ansible.builtin.assert: + that: + - (secrets_yaml.version | default('1.0')) is version('2.0', '>=') + fail_msg: >- + Bootstrap secret loading (secrets_phase=early) requires values-secret format version 2.0 or newer. + success_msg: values-secret version is 2.0 or newer; bootstrap (early) loading is allowed. + when: secrets_phase | default('late') == 'early' + + - name: Select Kubernetes secrets loader for bootstrap (early) phase + ansible.builtin.set_fact: + secrets_role: 'k8s_secret_utils' + tasks_from: 'inject_k8s_secrets' + when: + - secrets_phase | default('late') == 'early' + - (secrets_yaml.version | default('2.0')) is version('2.0', '>=') + + - name: Select Kubernetes secrets loader for late phase (kubernetes or none backend) ansible.builtin.set_fact: secrets_role: 'k8s_secret_utils' tasks_from: 'inject_k8s_secrets' when: + - secrets_phase | default('late') != 'early' - secrets_backing_store == "kubernetes" or secrets_backing_store == "none" - - secrets_yaml['version'] | default('2.0') >= '2.0' + - (secrets_yaml.version | default('2.0')) is version('2.0', '>=') # secrets_role will have been changed from the default if needed - name: Load secrets using designated role and tasks @@ -49,3 +78,4 @@ vault_policies: "{{ secrets_results['vault_policies'] }}" parsed_secrets: "{{ secrets_results['parsed_secrets'] }}" unique_vault_prefixes: "{{ secrets_results['unique_vault_prefixes'] | default([]) }}" + secrets_install_phase: "{{ secrets_phase | default('late') }}" diff --git a/plugins/module_utils/load_secrets_v2.py b/plugins/module_utils/load_secrets_v2.py index 90f1c19..1fd49ba 100644 --- a/plugins/module_utils/load_secrets_v2.py +++ b/plugins/module_utils/load_secrets_v2.py @@ -14,7 +14,10 @@ # under the License. """ -Module that implements V2 of the values-secret.yaml spec +Module that implements V2 of the values-secret.yaml spec. + +vault_load_secrets writes only the top-level ``secrets`` list to Vault; ``bootstrap_secrets`` are +validated but never pushed to Vault (bootstrap targets Kubernetes via the early none injector). """ from __future__ import absolute_import, division, print_function @@ -75,41 +78,52 @@ def _get_backingstore(self): """ return str(self.syaml.get("backingStore", "vault")) - def _get_secrets(self): - return self.syaml.get("secrets", {}) + def _get_bootstrap_secrets(self): + bootstrap = self.syaml.get("bootstrap_secrets", []) + if bootstrap is None or bootstrap == "None": + return [] + return bootstrap def _validate_secrets(self): secrets = self._get_secrets() - if len(secrets) == 0: + bootstrap_secrets = self._get_bootstrap_secrets() + if len(secrets) == 0 and len(bootstrap_secrets) == 0: self.module.fail_json("No secrets found") - # Validate each secret and collect names for duplicate checking + bootstrap_names = [] + for secret in bootstrap_secrets: + result = self._validate_secret(secret, is_bootstrap=True) + if not result[0]: + return result + bootstrap_names.append(secret["name"]) + secret_names = [] for secret in secrets: - result = self._validate_secret(secret) + result = self._validate_secret(secret, is_bootstrap=False) if not result[0]: return result secret_names.append(secret["name"]) - # Check for duplicate secret names - dupes = find_dupes(secret_names) - if len(dupes) > 0: - return (False, f"You cannot have duplicate secret names: {dupes}") + dupes_bootstrap = find_dupes(bootstrap_names) + if len(dupes_bootstrap) > 0: + return (False, f"You cannot have duplicate secret names in bootstrap_secrets: {dupes_bootstrap}") + dupes_secrets = find_dupes(secret_names) + if len(dupes_secrets) > 0: + return (False, f"You cannot have duplicate secret names in secrets: {dupes_secrets}") return (True, "") - def _validate_secret(self, secret): + def _validate_secret(self, secret, is_bootstrap=False): """Validate a single secret configuration""" # Check mandatory fields if "name" not in secret: return (False, f"Secret {secret} is missing name") - secret_name = secret["name"] - - # Validate vault prefixes - result = self._validate_vault_prefixes(secret) - if not result[0]: - return result + # Bootstrap secrets are only applied via early K8s (none) load in phased installs; vaultPrefixes are not used. + if not is_bootstrap: + result = self._validate_vault_prefixes(secret) + if not result[0]: + return result # Validate fields result = self._validate_secret_fields(secret) @@ -298,10 +312,26 @@ def inject_secrets(self): # This must come first as some passwords might depend on vault policies to exist. # It is a noop when no policies are defined self.inject_vault_policies() + bootstrap_secrets = self._get_bootstrap_secrets() secrets = self._get_secrets() + bootstrap_names = {s.get("name") for s in bootstrap_secrets if s.get("name")} + secrets_filtered = [s for s in secrets if s.get("name") not in bootstrap_names] + skipped = {s.get("name") for s in secrets if s.get("name") in bootstrap_names} + for n in sorted(skipped): + self.module.warn( + "Omitting secrets entry %r because it duplicates a bootstrap_secrets name " + "(bootstrap material is not written to Vault from this module)." % (n,) + ) + + if bootstrap_secrets: + self.module.warn( + "bootstrap_secrets are validated but not written to Vault; Vault is not used " + "for bootstrap material. Apply bootstrap secrets with the early-phase Kubernetes " + "(none) injector (for example load_bootstrap_secrets)." + ) - total_secrets = 0 # Counter for all the secrets uploaded - for s in secrets: + total_secrets = 0 # Counter for all the secrets uploaded (secrets[] only; never bootstrap_secrets) + for s in secrets_filtered: counter = 0 # This counter is to use kv put on first secret and kv patch on latter sname = s.get("name") fields = s.get("fields", []) diff --git a/plugins/module_utils/parse_secrets_v2.py b/plugins/module_utils/parse_secrets_v2.py index 960de5b..2ccd1c1 100644 --- a/plugins/module_utils/parse_secrets_v2.py +++ b/plugins/module_utils/parse_secrets_v2.py @@ -36,13 +36,16 @@ class ParseSecretsV2(SecretsV2Base): - def __init__(self, module, syaml, secrets_backing_store): + def __init__(self, module, syaml, secrets_backing_store, secrets_phase="late"): super().__init__(module, syaml) self.secrets_backing_store = str(secrets_backing_store) + self.secrets_phase = secrets_phase self.secret_store_namespace = None self.parsed_secrets = {} self.kubernetes_secret_objects = [] self.vault_policies = {} + # Set during parse(): backing store semantics for the current phase (early -> none). + self._parse_backing_store = None def _get_backingstore(self): """ @@ -88,6 +91,42 @@ def _get_secrets(self): # We also check for None here to cover when there is no jinja filter is used (unit tests) return [] if secrets == "None" or secrets is None else secrets + def _get_bootstrap_secrets(self): + bootstrap = self.syaml.get("bootstrap_secrets", []) + return [] if bootstrap == "None" or bootstrap is None else bootstrap + + def _late_secrets_excluding_bootstrap(self): + """ + Late phase must not re-apply secrets that are defined under bootstrap_secrets + (those are only parsed/injected in the early phase). + """ + bootstrap_names = { + s.get("name") for s in self._get_bootstrap_secrets() if s.get("name") + } + skipped = set() + out = [] + for s in self._get_secrets(): + n = s.get("name") + if n in bootstrap_names: + skipped.add(n) + else: + out.append(s) + for n in sorted(skipped): + self.module.warn( + "Late-phase secrets: omitting secrets entry %r because that name is listed under " + "bootstrap_secrets (bootstrap secrets are only applied in the early phase)." % (n,) + ) + return out + + def _secrets_for_phase(self): + if self.secrets_phase == "early": + return self._get_bootstrap_secrets() + if self.secrets_phase == "late": + return self._late_secrets_excluding_bootstrap() + self.module.fail_json( + f"secrets_phase must be 'early' or 'late', not {self.secrets_phase!r}" + ) + def _get_field_annotations(self, f): return f.get("annotations", {}) @@ -150,126 +189,175 @@ def _create_k8s_secret(self, sname, secret_type, namespace, labels, annotations) "stringData": {}, } + def _backing_store_for_parse(self): + """ + Effective backing store for the current parse run. + + Bootstrap (early) secrets are always parsed with the 'none' backend so they + resolve to Kubernetes secret objects and never use vault-only features. + """ + if self.secrets_phase == "early": + return "none" + return self._get_backingstore() + # This does what inject_secrets used to (mostly) def parse(self): self.sanitize_values() self.vault_policies = self._get_vault_policies() self.secret_store_namespace = self._get_secret_store_namespace() - backing_store = self._get_backingstore() - secrets = self._get_secrets() + backing_store = self._backing_store_for_parse() + self._parse_backing_store = backing_store + secrets = self._secrets_for_phase() total_secrets = 0 # Counter for all the secrets uploaded if len(secrets) == 0: self.module.warn("No secrets were parsed") + self._parse_backing_store = None return total_secrets - for s in secrets: - total_secrets += 1 - counter = 0 # This counter is to use kv put on first secret and kv patch on latter - sname = s.get("name") - fields = s.get("fields", []) - vault_prefixes = self._get_vault_prefixes(s) - secret_type = s.get("type", "Opaque") - vault_mount = s.get("vaultMount", "secret") - target_namespaces = s.get("targetNamespaces", []) - labels = stringify_dict(s.get("labels", self._get_default_labels())) - annotations = stringify_dict( - s.get("annotations", self._get_default_annotations()) - ) + try: + for s in secrets: + total_secrets += 1 + counter = 0 # This counter is to use kv put on first secret and kv patch on latter + sname = s.get("name") + fields = s.get("fields", []) + vault_prefixes = ( + [] if backing_store == "none" else self._get_vault_prefixes(s) + ) + secret_type = s.get("type", "Opaque") + vault_mount = s.get("vaultMount", "secret") + target_namespaces = s.get("targetNamespaces", []) + labels = stringify_dict(s.get("labels", self._get_default_labels())) + annotations = stringify_dict( + s.get("annotations", self._get_default_annotations()) + ) - self.parsed_secrets[sname] = { - "name": sname, - "fields": {}, - "vault_mount": vault_mount, - "vault_policies": {}, - "vault_prefixes": vault_prefixes, - "override": [], - "generate": [], - "paths": {}, - "base64": [], - "ini_file": {}, - "type": secret_type, - "target_namespaces": target_namespaces, - "labels": labels, - "annotations": annotations, - } + self.parsed_secrets[sname] = { + "name": sname, + "fields": {}, + "vault_mount": vault_mount, + "vault_policies": {}, + "vault_prefixes": vault_prefixes, + "override": [], + "generate": [], + "paths": {}, + "base64": [], + "ini_file": {}, + "type": secret_type, + "target_namespaces": target_namespaces, + "labels": labels, + "annotations": annotations, + } + + for i in fields: + self._inject_field(sname, i) + counter += 1 + + if backing_store == "kubernetes": + k8s_namespaces = [self._get_secret_store_namespace()] + elif backing_store == "vault": + # Vault injector does not materialize Kubernetes secrets; ignore targetNamespaces. + k8s_namespaces = [] + else: + k8s_namespaces = target_namespaces - for i in fields: - self._inject_field(sname, i) - counter += 1 + for tns in k8s_namespaces: + k8s_secret = self._create_k8s_secret( + sname, secret_type, tns, labels, annotations + ) + k8s_secret["stringData"] = self.parsed_secrets[sname]["fields"] + self.kubernetes_secret_objects.append(k8s_secret) - if backing_store == "kubernetes": - k8s_namespaces = [self._get_secret_store_namespace()] - else: - k8s_namespaces = target_namespaces + return total_secrets + finally: + self._parse_backing_store = None + + def _effective_backing_store_for_field_ops(self): + return ( + self._parse_backing_store + if self._parse_backing_store is not None + else self._get_backingstore() + ) + + def _validate_one_secret_entry(self, s, backing_store): + if "name" not in s: + return (False, "Secret entry is missing name") + + vault_prefixes = s.get("vaultPrefixes", ["hub"]) + # Vault path prefixes are not used for the 'none' backend (early bootstrap and none-backed K8s inject). + if backing_store != "none": + if vault_prefixes is None or len(vault_prefixes) == 0: + return (False, f"Secret {s['name']} has empty vaultPrefixes") - for tns in k8s_namespaces: - k8s_secret = self._create_k8s_secret( - sname, secret_type, tns, labels, annotations + namespaces = s.get("targetNamespaces", []) + if not isinstance(namespaces, list): + return (False, f"Secret {s['name']} targetNamespaces must be a list") + + if backing_store == "none" and len(namespaces) == 0: + return ( + False, + f"Secret {s['name']} targetNamespaces cannot be empty for secrets backend {backing_store}", + ) # noqa: E501 + + labels = s.get("labels", {}) + if not isinstance(labels, dict): + return (False, f"Secret {s['name']} labels must be a dictionary") + + annotations = s.get("annotations", {}) + if not isinstance(annotations, dict): + return (False, f"Secret {s['name']} annotations must be a dictionary") + + fields = s.get("fields", []) + if len(fields) == 0: + return (False, f"Secret {s['name']} does not have any fields") + + field_names = [] + for i in fields: + (ret, msg) = self._validate_field(i) + if not ret: + return (False, msg) + if backing_store == "none" and self._get_field_on_missing_value(i) == "generate": + return ( + False, + f"Secret {s['name']} field {i['name']} cannot use onMissingValue generate " + "with secrets backend none", ) - k8s_secret["stringData"] = self.parsed_secrets[sname]["fields"] - self.kubernetes_secret_objects.append(k8s_secret) + field_names.append(i["name"]) + field_dupes = find_dupes(field_names) + if len(field_dupes) > 0: + return (False, f"You cannot have duplicate field names: {field_dupes}") - return total_secrets + return (True, "") def _validate_secrets(self): backing_store = self._get_backingstore() secrets = self._get_secrets() - if len(secrets) == 0: + bootstrap_secrets = self._get_bootstrap_secrets() + if len(secrets) == 0 and len(bootstrap_secrets) == 0: self.module.warn("No secrets found") return (True, "") - names = [] - for s in secrets: - # These fields are mandatory - for i in ["name"]: - try: - unused = s[i] - except KeyError: - return (False, f"Secret {s['name']} is missing {i}") - names.append(s["name"]) - - vault_prefixes = s.get("vaultPrefixes", ["hub"]) - # This checks for the case when vaultPrefixes: is specified but empty - if vault_prefixes is None or len(vault_prefixes) == 0: - return (False, f"Secret {s['name']} has empty vaultPrefixes") + bootstrap_names = [] + for s in bootstrap_secrets: + (ret, msg) = self._validate_one_secret_entry(s, "none") + if not ret: + return (False, msg) + bootstrap_names.append(s["name"]) - namespaces = s.get("targetNamespaces", []) - if not isinstance(namespaces, list): - return (False, f"Secret {s['name']} targetNamespaces must be a list") - - if backing_store == "none" and namespaces == []: - return ( - False, - f"Secret {s['name']} targetNamespaces cannot be empty for secrets backend {backing_store}", - ) # noqa: E501 - - labels = s.get("labels", {}) - if not isinstance(labels, dict): - return (False, f"Secret {s['name']} labels must be a dictionary") - - annotations = s.get("annotations", {}) - if not isinstance(annotations, dict): - return (False, f"Secret {s['name']} annotations must be a dictionary") - - fields = s.get("fields", []) - if len(fields) == 0: - return (False, f"Secret {s['name']} does not have any fields") - - field_names = [] - for i in fields: - (ret, msg) = self._validate_field(i) - if not ret: - return (False, msg) - field_names.append(i["name"]) - field_dupes = find_dupes(field_names) - if len(field_dupes) > 0: - return (False, f"You cannot have duplicate field names: {field_dupes}") - - dupes = find_dupes(names) - if len(dupes) > 0: - return (False, f"You cannot have duplicate secret names: {dupes}") + secret_names = [] + for s in secrets: + (ret, msg) = self._validate_one_secret_entry(s, backing_store) + if not ret: + return (False, msg) + secret_names.append(s["name"]) + + dupes_bootstrap = find_dupes(bootstrap_names) + if len(dupes_bootstrap) > 0: + return (False, f"You cannot have duplicate secret names in bootstrap_secrets: {dupes_bootstrap}") + dupes_secrets = find_dupes(secret_names) + if len(dupes_secrets) > 0: + return (False, f"You cannot have duplicate secret names in secrets: {dupes_secrets}") return (True, "") def sanitize_values(self): @@ -285,6 +373,11 @@ def sanitize_values(self): if v not in ["2.0"]: self.module.fail_json(f"Version is not 2.0: {v}") + if self.secrets_phase not in ("early", "late"): + self.module.fail_json( + f"secrets_phase must be 'early' or 'late', not {self.secrets_phase!r}" + ) + backing_store = self._get_backingstore() if backing_store not in [ "kubernetes", @@ -314,7 +407,7 @@ def _inject_field(self, secret_name, f): if kind in ["value", ""]: if on_missing_value == "generate": self.parsed_secrets[secret_name]["generate"].append(f["name"]) - if self._get_backingstore() != "vault": + if self._effective_backing_store_for_field_ops() != "vault": self.module.fail_json( "You cannot have onMissingValue set to 'generate' unless using vault backingstore " f"for secret {secret_name} field {f['name']}" diff --git a/plugins/modules/parse_secrets_info.py b/plugins/modules/parse_secrets_info.py index 0097ac6..8001821 100644 --- a/plugins/modules/parse_secrets_info.py +++ b/plugins/modules/parse_secrets_info.py @@ -81,10 +81,23 @@ type: str secrets_backing_store: description: - - The secrets backing store that will be used for parsed secrets + - The secrets backing store that will be used for parsed secrets. + - When C(vault), C(kubernetes_secret_objects) is always empty for the late-phase Vault injector; + C(targetNamespaces) on C(secrets) entries are not used to build Kubernetes objects. required: false default: vault type: str + secrets_phase: + description: + - Which phase of secrets to parse. When set to C(early), C(bootstrap_secrets) are parsed using the C(none) + backend semantics (Kubernetes secret objects, no vault-only features) regardless of + C(secrets_backing_store); C(vaultPrefixes) on those entries are not used. When set to C(late), entries under C(secrets) use C(secrets_backing_store), + excluding any entry whose C(name) matches a secret in C(bootstrap_secrets) (those are only applied in the early phase). + The full file is still validated on every call. + required: false + default: late + type: str + choices: [early, late] """ RETURN = """ @@ -102,11 +115,17 @@ secrets_backing_store: 'kubernetes' register: secrets_info -- name: Parse secrets file into data structures +- name: Parse secrets file (none backing store) parse_secrets_info: values_secrets_plaintext: '{{ }}' secrets_backing_store: 'none' register: secrets_info + +- name: Parse secrets file for early phase (bootstrap_secrets only) + parse_secrets_info: + values_secrets_plaintext: '{{ }}' + secrets_phase: early + register: secrets_info_early """ import traceback @@ -136,13 +155,14 @@ def run(module): args = module.params values_secrets_plaintext = args.get("values_secrets_plaintext", "") secrets_backing_store = args.get("secrets_backing_store", "vault") + secrets_phase = args.get("secrets_phase", "late") syaml = yaml.safe_load(values_secrets_plaintext) if syaml is None: syaml = {} - parsed_secret_obj = ParseSecretsV2(module, syaml, secrets_backing_store) + parsed_secret_obj = ParseSecretsV2(module, syaml, secrets_backing_store, secrets_phase) parsed_secret_obj.parse() results["failed"] = False diff --git a/plugins/modules/vault_load_secrets.py b/plugins/modules/vault_load_secrets.py index 73544fd..1e8bc05 100644 --- a/plugins/modules/vault_load_secrets.py +++ b/plugins/modules/vault_load_secrets.py @@ -69,7 +69,10 @@ - Michele Baldessari (@mbaldess) - Martin Jackson (@mhjacks) description: - - Takes a values-secret.yaml file and uploads the secrets into the HashiCorp Vault + - Takes a values-secret.yaml file and uploads secrets into the HashiCorp Vault. + - For format version 2.0, only the top-level C(secrets) list is written to Vault. C(bootstrap_secrets) + are validated with the same file rules but are never pushed to Vault (Vault is not available for + bootstrap material; use the early-phase Kubernetes C(none) injector instead). options: values_secrets: description: diff --git a/roles/k8s_secret_utils/defaults/main.yml b/roles/k8s_secret_utils/defaults/main.yml index 7ebda20..f080f1c 100644 --- a/roles/k8s_secret_utils/defaults/main.yml +++ b/roles/k8s_secret_utils/defaults/main.yml @@ -1,2 +1,4 @@ --- +# secrets_install_phase: early|late — set by load_secrets/process_secrets include_role vars, or by parse_secrets in this role. +# bootstrap_phase_*_changed: counts for early-phase summary (set during inject_k8s_secrets; not secret material). secrets_ns: 'validated-patterns-secrets' diff --git a/roles/k8s_secret_utils/tasks/ensure_one_bootstrap_namespace.yml b/roles/k8s_secret_utils/tasks/ensure_one_bootstrap_namespace.yml new file mode 100644 index 0000000..5ba01a8 --- /dev/null +++ b/roles/k8s_secret_utils/tasks/ensure_one_bootstrap_namespace.yml @@ -0,0 +1,28 @@ +--- +# Early-phase bootstrap only: create Namespace only when absent (never replace an existing NS). +- name: Look up namespace {{ bootstrap_target_namespace }} + kubernetes.core.k8s_info: + kind: Namespace + name: "{{ bootstrap_target_namespace }}" + register: _bootstrap_ns_info + changed_when: false + +- name: Create namespace {{ bootstrap_target_namespace }} when absent + when: _bootstrap_ns_info.resources | length == 0 + kubernetes.core.k8s: + state: present + definition: + apiVersion: v1 + kind: Namespace + metadata: + name: "{{ bootstrap_target_namespace }}" + register: _bootstrap_ns_create_result + +- name: Record bootstrap namespace change count + ansible.builtin.set_fact: + bootstrap_phase_namespaces_changed: "{{ (bootstrap_phase_namespaces_changed | default(0)) | int + 1 }}" + when: + - (secrets_install_phase | default(secrets_phase | default('late'))) == 'early' + - _bootstrap_ns_create_result is defined + - not (_bootstrap_ns_create_result.skipped | default(false)) + - _bootstrap_ns_create_result.changed | default(false) diff --git a/roles/k8s_secret_utils/tasks/inject_k8s_secret.yml b/roles/k8s_secret_utils/tasks/inject_k8s_secret.yml index 410e1a0..5558a9b 100644 --- a/roles/k8s_secret_utils/tasks/inject_k8s_secret.yml +++ b/roles/k8s_secret_utils/tasks/inject_k8s_secret.yml @@ -1,15 +1,26 @@ --- -- name: Check for secrets namespace - no_log: '{{ hide_sensitive_output | default(true) }}' +- name: Wait for namespace {{ _k8s_secret_object.metadata.namespace }} (secret {{ _k8s_secret_object.metadata.name }}) + when: (secrets_install_phase | default(secrets_phase | default('late'))) != 'early' kubernetes.core.k8s_info: kind: Namespace - name: "{{ item['metadata']['namespace'] }}" + name: "{{ _k8s_secret_object.metadata.namespace }}" register: secrets_ns_rc until: secrets_ns_rc.resources | length > 0 retries: 20 delay: 45 + changed_when: false -- name: Inject k8s secret +- name: Inject Kubernetes secret {{ _k8s_secret_object.metadata.namespace }}/{{ _k8s_secret_object.metadata.name }} no_log: '{{ hide_sensitive_output | default(True) }}' kubernetes.core.k8s: - definition: '{{ item }}' + definition: '{{ _k8s_secret_object }}' + register: _k8s_secret_apply_result + +- name: Record bootstrap secret change count + ansible.builtin.set_fact: + bootstrap_phase_secrets_changed: "{{ (bootstrap_phase_secrets_changed | default(0)) | int + 1 }}" + when: + - (secrets_install_phase | default(secrets_phase | default('late'))) == 'early' + - _k8s_secret_apply_result is defined + - not (_k8s_secret_apply_result.skipped | default(false)) + - _k8s_secret_apply_result.changed | default(false) diff --git a/roles/k8s_secret_utils/tasks/inject_k8s_secrets.yml b/roles/k8s_secret_utils/tasks/inject_k8s_secrets.yml index fab658f..1b26c61 100644 --- a/roles/k8s_secret_utils/tasks/inject_k8s_secrets.yml +++ b/roles/k8s_secret_utils/tasks/inject_k8s_secrets.yml @@ -1,5 +1,40 @@ --- -- name: Inject secrets - no_log: '{{ hide_sensitive_output | default(True) }}' +# Early phase only: create each bootstrap secret target namespace if absent (never replace an existing NS). +# Late phase expects namespaces from the pattern/operator; inject_k8s_secret.yml waits until they exist. +# secrets_install_phase is passed from load_secrets/process_secrets or set in parse_secrets for this role. +- name: Initialize bootstrap phase change counters for this inject run + ansible.builtin.set_fact: + bootstrap_phase_namespaces_changed: 0 + bootstrap_phase_secrets_changed: 0 + when: (secrets_install_phase | default(secrets_phase | default('late'))) == 'early' + +- name: Clear bootstrap phase counters before late-phase inject + ansible.builtin.set_fact: + bootstrap_phase_namespaces_changed: 0 + bootstrap_phase_secrets_changed: 0 + when: (secrets_install_phase | default(secrets_phase | default('late'))) != 'early' + +- name: Create missing bootstrap target namespaces + ansible.builtin.include_tasks: ensure_one_bootstrap_namespace.yml + loop: "{{ kubernetes_secret_objects | map(attribute='metadata') | map(attribute='namespace') | unique | list }}" + loop_control: + loop_var: bootstrap_target_namespace + label: "{{ bootstrap_target_namespace }}" + when: + - (secrets_install_phase | default(secrets_phase | default('late'))) == 'early' + - kubernetes_secret_objects | length > 0 + +- name: Inject Kubernetes secrets ansible.builtin.include_tasks: inject_k8s_secret.yml loop: '{{ kubernetes_secret_objects }}' + loop_control: + loop_var: _k8s_secret_object + label: "{{ _k8s_secret_object.metadata.namespace }}/{{ _k8s_secret_object.metadata.name }}" + +- name: Report bootstrap phase Kubernetes apply summary (counts only) + ansible.builtin.debug: + msg: >- + Bootstrap phase: {{ bootstrap_phase_namespaces_changed | default(0) }} namespace(s) created or updated, + {{ bootstrap_phase_secrets_changed | default(0) }} secret(s) created or updated in the cluster. + when: (secrets_install_phase | default(secrets_phase | default('late'))) == 'early' + changed_when: false diff --git a/roles/k8s_secret_utils/tasks/parse_secrets.yml b/roles/k8s_secret_utils/tasks/parse_secrets.yml index 2fa4cb2..830e373 100644 --- a/roles/k8s_secret_utils/tasks/parse_secrets.yml +++ b/roles/k8s_secret_utils/tasks/parse_secrets.yml @@ -4,8 +4,13 @@ parse_secrets_info: values_secrets_plaintext: "{{ values_secrets_data }}" secrets_backing_store: "{{ secrets_backing_store }}" + secrets_phase: "{{ secrets_phase | default('late') }}" register: secrets_results +- name: Record secrets install phase for inject tasks + ansible.builtin.set_fact: + secrets_install_phase: "{{ secrets_phase | default('late') }}" + - name: Return kubernetes objects no_log: '{{ hide_sensitive_output | default(true) }}' ansible.builtin.set_fact: diff --git a/roles/load_secrets/defaults/main.yml b/roles/load_secrets/defaults/main.yml index 0999947..39b1f37 100644 --- a/roles/load_secrets/defaults/main.yml +++ b/roles/load_secrets/defaults/main.yml @@ -1,3 +1,8 @@ secrets_role: "vault_utils" tasks_from: "push_parsed_secrets" hide_sensitive_output: true +# late: parse entries under secrets. early: parse bootstrap_secrets only (full file is still validated). +secrets_phase: late +# Set by load_secrets after early-phase apply or late-phase bootstrap preflight; avoids re-applying +# bootstrap_secrets when load_bootstrap_secrets.yml already ran in this playbook (localhost). +# secrets_install_phase is set on the host by this role before invoking the backend role (k8s injector reads it). diff --git a/roles/load_secrets/tasks/ensure_bootstrap_before_late.yml b/roles/load_secrets/tasks/ensure_bootstrap_before_late.yml new file mode 100644 index 0000000..1906604 --- /dev/null +++ b/roles/load_secrets/tasks/ensure_bootstrap_before_late.yml @@ -0,0 +1,27 @@ +--- +# Late phase only: if load_bootstrap_secrets (or another early apply) did not run earlier +# on this localhost in the same playbook, apply bootstrap_secrets once via the early +# Kubernetes path before the main late parse (which omits bootstrap from secrets[]). +- name: Parse bootstrap secrets for Kubernetes injection before late-phase parse + no_log: "{{ hide_sensitive_output | default(true) }}" + parse_secrets_info: + values_secrets_plaintext: "{{ values_secrets_data }}" + secrets_backing_store: "{{ secrets_backing_store }}" + secrets_phase: early + register: _bootstrap_parse_for_late + +- name: Inject bootstrap Kubernetes secrets before late phase + ansible.builtin.include_role: + name: k8s_secret_utils + tasks_from: inject_k8s_secrets + vars: + kubernetes_secret_objects: "{{ _bootstrap_parse_for_late.kubernetes_secret_objects }}" + vault_policies: "{{ _bootstrap_parse_for_late.vault_policies }}" + parsed_secrets: "{{ _bootstrap_parse_for_late.parsed_secrets }}" + unique_vault_prefixes: "{{ _bootstrap_parse_for_late.unique_vault_prefixes | default([]) }}" + secrets_install_phase: early + when: _bootstrap_parse_for_late.kubernetes_secret_objects | length > 0 + +- name: Mark bootstrap secrets as loaded for this playbook run + ansible.builtin.set_fact: + bootstrap_secrets_loaded_this_run: true diff --git a/roles/load_secrets/tasks/main.yml b/roles/load_secrets/tasks/main.yml index 7d79b09..6e842bd 100644 --- a/roles/load_secrets/tasks/main.yml +++ b/roles/load_secrets/tasks/main.yml @@ -1,4 +1,11 @@ --- +# set_fact for secrets_role/tasks_from in an early run persists on localhost for the whole +# playbook; reset each time this role runs so late vault path is not skipped after bootstrap. +- name: Reset secrets loader selection for this load_secrets invocation + ansible.builtin.set_fact: + secrets_role: "vault_utils" + tasks_from: "push_parsed_secrets" + - name: Set fact for secretStore backend ansible.builtin.set_fact: secrets_backing_store: "{{ values_global.global.secretStore.backend | default('vault') }}" @@ -10,31 +17,58 @@ - cluster_pre_check - find_vp_secrets -- name: Fail if values_secrets_data is missing - ansible.builtin.shell: | - printf "ERROR\n" - printf " values_secrets_data was not found.\n" - printf " The find_vp_secrets role should set it.\n" - printf " Ensure your values/secret files are present and readable.\n" - exit 1 - when: values_secrets_data is not defined +- name: Assert values_secrets_data is present after find_vp_secrets + ansible.builtin.assert: + that: + - values_secrets_data is defined + fail_msg: >- + values_secrets_data was not found. The find_vp_secrets role should set it. + Ensure your values/secret files are present and readable. + success_msg: values_secrets_data is defined; continuing with secret loading. - name: Determine how to load secrets ansible.builtin.set_fact: secrets_yaml: "{{ values_secrets_data if values_secrets_data is not string else values_secrets_data | from_yaml }}" +- name: Ensure bootstrap secrets before late phase when not already applied this run + ansible.builtin.include_tasks: ensure_bootstrap_before_late.yml + when: + - secrets_phase | default('late') == 'late' + - not (bootstrap_secrets_loaded_this_run | default(false)) + - (secrets_yaml.version | default('2.0')) is version('2.0', '>=') + - (secrets_yaml.bootstrap_secrets | default([]) | length) > 0 + - name: Parse secrets data no_log: "{{ hide_sensitive_output | default(true) }}" parse_secrets_info: values_secrets_plaintext: "{{ values_secrets_data }}" secrets_backing_store: "{{ secrets_backing_store }}" + secrets_phase: "{{ secrets_phase }}" register: secrets_results -- name: Select Kubernetes secrets loader (when requested) +- name: Assert values-secret v2 when loading bootstrap (early) secrets + ansible.builtin.assert: + that: + - (secrets_yaml.version | default('1.0')) is version('2.0', '>=') + fail_msg: >- + Bootstrap secret loading (secrets_phase=early) requires values-secret format version 2.0 or newer. + success_msg: values-secret version is 2.0 or newer; bootstrap (early) loading is allowed. + when: secrets_phase | default('late') == 'early' + +- name: Select Kubernetes secrets loader for bootstrap (early) phase + ansible.builtin.set_fact: + secrets_role: "k8s_secret_utils" + tasks_from: "inject_k8s_secrets" + when: + - secrets_phase | default('late') == 'early' + - (secrets_yaml.version | default('2.0')) is version('2.0', '>=') + +- name: Select Kubernetes secrets loader for late phase (kubernetes or none backend) ansible.builtin.set_fact: secrets_role: "k8s_secret_utils" tasks_from: "inject_k8s_secrets" when: + - secrets_phase | default('late') != 'early' - secrets_backing_store in ["kubernetes", "none"] - (secrets_yaml.version | default('2.0')) is version('2.0', '>=') @@ -46,4 +80,12 @@ kubernetes_secret_objects: "{{ secrets_results.kubernetes_secret_objects }}" vault_policies: "{{ secrets_results.vault_policies }}" parsed_secrets: "{{ secrets_results.parsed_secrets }}" - unique_vault_prefixes: "{{ secrets_results.unique_vault_prefixes }}" + unique_vault_prefixes: "{{ secrets_results.unique_vault_prefixes | default([]) }}" + secrets_install_phase: "{{ secrets_phase | default('late') }}" + +- name: Mark bootstrap secrets as loaded this playbook run after early-phase apply + ansible.builtin.set_fact: + bootstrap_secrets_loaded_this_run: true + when: + - secrets_phase | default('late') == 'early' + - (secrets_yaml.version | default('2.0')) is version('2.0', '>=') diff --git a/roles/vault_utils/values-secrets.v2.schema.json b/roles/vault_utils/values-secrets.v2.schema.json index b5582c3..81a9221 100644 --- a/roles/vault_utils/values-secrets.v2.schema.json +++ b/roles/vault_utils/values-secrets.v2.schema.json @@ -120,12 +120,13 @@ }, "secrets": { "$ref": "#/definitions/Secrets", - "description": "The list of actual secrets to be uploaded in the vault" + "description": "Secrets loaded in the late phase; entries whose name matches bootstrap_secrets are omitted (bootstrap is early-only). See bootstrap_secrets for the early phase." + }, + "bootstrap_secrets": { + "$ref": "#/definitions/BootstrapSecrets", + "description": "Early-phase secrets; parsed with the none backend (Kubernetes targets via targetNamespaces, no vault generate fields)" } }, - "required": [ - "secrets" - ], "title": "Values Secrets V2 Format" }, "VaultPolicies": { @@ -154,6 +155,44 @@ "$ref": "#/definitions/Secret" } }, + "BootstrapSecrets": { + "type": "array", + "description": "Early-phase secrets (parsed with none backend; each item needs targetNamespaces)", + "items": { + "$ref": "#/definitions/BootstrapSecret" + } + }, + "BootstrapSecret": { + "allOf": [ + { + "$ref": "#/definitions/Secret" + }, + { + "type": "object", + "required": ["targetNamespaces"], + "properties": { + "targetNamespaces": { + "type": "array", + "minItems": 1, + "items": { + "type": "string", + "minLength": 1 + }, + "uniqueItems": true + }, + "vaultPrefixes": { + "type": "array", + "description": "Not used for bootstrap: those entries are applied only via the early-phase Kubernetes (none) injector, not Vault.", + "items": { + "type": "string", + "minLength": 1, + "uniqueItems": true + } + } + } + } + ] + }, "Secret": { "type": "object", "description": "The single secret to be injected into the vault", @@ -181,7 +220,7 @@ }, "targetNamespaces": { "type": "array", - "description": "The namespace(s) that the secret will be injected into, ignored by configs using ESO", + "description": "Namespaces for Kubernetes secret injection (kubernetes or none backingStore). When backingStore is vault, late-phase Vault loading does not use these for Kubernetes objects.", "items": { "type": "string", "minItems": 1, diff --git a/tests/unit/test_parse_secrets.py b/tests/unit/test_parse_secrets.py index 2d10455..9aca5be 100644 --- a/tests/unit/test_parse_secrets.py +++ b/tests/unit/test_parse_secrets.py @@ -871,7 +871,8 @@ def test_ensure_error_secrets_same_name(self, getpass): ret = ansible_err.exception.args[0] self.assertEqual(ret["failed"], True) assert ( - ret["args"][1] == "You cannot have duplicate secret names: ['config-demo']" + ret["args"][1] + == "You cannot have duplicate secret names in secrets: ['config-demo']" ) def test_ensure_error_fields_same_name(self, getpass): @@ -927,7 +928,7 @@ def test_ensure_generate_errors_on_none_generate(self, getpass): self.assertEqual(ret["failed"], True) assert ( ret["args"][1] - == "You cannot have onMissingValue set to 'generate' unless using vault backingstore for secret config-demo field secret" # noqa: E501 + == "Secret config-demo field secret cannot use onMissingValue generate with secrets backend none" # noqa: E501 ) def test_ensure_success_empty_secrets(self, getpass): @@ -972,6 +973,315 @@ def test_ensure_success_null_secrets(self, getpass): and (len(ret["kubernetes_secret_objects"]) == 0) ) + def test_bootstrap_early_phase_parses_bootstrap_only(self, getpass): + testfile_output = self.get_file_as_stdout( + os.path.join(self.testdir_v2, "values-secret-v2-bootstrap-and-late.yaml") + ) + with self.assertRaises(AnsibleExitJson) as ansible_err: + set_module_args( + { + "values_secrets_plaintext": testfile_output, + "secrets_phase": "early", + } + ) + parse_secrets_info.main() + + ret = ansible_err.exception.args[0] + self.assertTrue(ret["failed"] is False) + self.assertEqual(set(ret["parsed_secrets"].keys()), {"bootstrap-only"}) + + def test_bootstrap_late_phase_parses_secrets_only(self, getpass): + testfile_output = self.get_file_as_stdout( + os.path.join(self.testdir_v2, "values-secret-v2-bootstrap-and-late.yaml") + ) + with self.assertRaises(AnsibleExitJson) as ansible_err: + set_module_args( + { + "values_secrets_plaintext": testfile_output, + "secrets_phase": "late", + } + ) + parse_secrets_info.main() + + ret = ansible_err.exception.args[0] + self.assertTrue(ret["failed"] is False) + self.assertEqual(set(ret["parsed_secrets"].keys()), {"late-only"}) + + def test_vault_late_parse_does_not_emit_k8s_secrets_for_target_namespaces(self, getpass): + yaml_content = """ +version: "2.0" +backingStore: vault +secrets: + - name: vault-only + targetNamespaces: + - documented-but-unused + vaultPrefixes: + - hub + fields: + - name: k + value: v + onMissingValue: error +""" + with self.assertRaises(AnsibleExitJson) as ansible_err: + set_module_args( + { + "values_secrets_plaintext": yaml_content, + "secrets_backing_store": "vault", + "secrets_phase": "late", + } + ) + parse_secrets_info.main() + + ret = ansible_err.exception.args[0] + self.assertEqual(ret["failed"], False) + self.assertEqual(len(ret["kubernetes_secret_objects"]), 0) + self.assertEqual( + ret["parsed_secrets"]["vault-only"]["target_namespaces"], + ["documented-but-unused"], + ) + + def test_bootstrap_only_file_early_vs_late(self, getpass): + testfile_output = self.get_file_as_stdout( + os.path.join(self.testdir_v2, "values-secret-v2-bootstrap-only.yaml") + ) + with self.assertRaises(AnsibleExitJson) as early_ret: + set_module_args( + { + "values_secrets_plaintext": testfile_output, + "secrets_phase": "early", + } + ) + parse_secrets_info.main() + er = early_ret.exception.args[0] + self.assertEqual(set(er["parsed_secrets"].keys()), {"only-bootstrap"}) + + with self.assertRaises(AnsibleExitJson) as late_ret: + set_module_args( + { + "values_secrets_plaintext": testfile_output, + "secrets_phase": "late", + } + ) + parse_secrets_info.main() + lr = late_ret.exception.args[0] + self.assertEqual(len(lr["parsed_secrets"]), 0) + + def test_bootstrap_secret_requires_target_namespaces(self, getpass): + testfile_output = self.get_file_as_stdout( + os.path.join( + self.testdir_v2, "values-secret-v2-bootstrap-missing-targetns.yaml" + ) + ) + with self.assertRaises(AnsibleFailJson) as ansible_err: + set_module_args( + { + "values_secrets_plaintext": testfile_output, + } + ) + parse_secrets_info.main() + + ret = ansible_err.exception.args[0] + self.assertEqual(ret["failed"], True) + assert ( + "boot-without-ns targetNamespaces cannot be empty for secrets backend none" + in ret["args"][1] + ) + + def test_bootstrap_early_rejects_generate_even_when_configured_backend_is_vault( + self, getpass + ): + yaml_content = """ +version: "2.0" +backingStore: vault +vaultPolicies: + basicPolicy: | + length=10 + rule "charset" { charset = "abcdefghijklmnopqrstuvwxyz" min-chars = 1 } +bootstrap_secrets: + - name: boot-gen + targetNamespaces: + - default + vaultPrefixes: + - hub + fields: + - name: s + onMissingValue: generate + vaultPolicy: basicPolicy +secrets: [] +""" + with self.assertRaises(AnsibleFailJson) as ansible_err: + set_module_args( + { + "values_secrets_plaintext": yaml_content, + "secrets_backing_store": "vault", + "secrets_phase": "early", + } + ) + parse_secrets_info.main() + + ret = ansible_err.exception.args[0] + self.assertEqual(ret["failed"], True) + assert "cannot use onMissingValue generate" in ret["args"][1] + assert "none" in ret["args"][1] + + def test_bootstrap_dup_name_across_sections_late_omits_secrets_copy(self, getpass): + testfile_output = self.get_file_as_stdout( + os.path.join(self.testdir_v2, "values-secret-v2-bootstrap-dup-across.yaml") + ) + with self.assertRaises(AnsibleExitJson) as ansible_err: + set_module_args( + { + "values_secrets_plaintext": testfile_output, + "secrets_phase": "late", + } + ) + parse_secrets_info.main() + + ret = ansible_err.exception.args[0] + self.assertEqual(ret["failed"], False) + self.assertEqual(len(ret["parsed_secrets"]), 0) + self.assertEqual(len(ret["kubernetes_secret_objects"]), 0) + + def test_bootstrap_dup_name_across_sections_early_parses_bootstrap_only(self, getpass): + testfile_output = self.get_file_as_stdout( + os.path.join(self.testdir_v2, "values-secret-v2-bootstrap-dup-across.yaml") + ) + with self.assertRaises(AnsibleExitJson) as ansible_err: + set_module_args( + { + "values_secrets_plaintext": testfile_output, + "secrets_phase": "early", + } + ) + parse_secrets_info.main() + + ret = ansible_err.exception.args[0] + self.assertEqual(ret["failed"], False) + self.assertEqual(set(ret["parsed_secrets"].keys()), {"config-demo"}) + self.assertEqual(ret["parsed_secrets"]["config-demo"]["fields"]["a"], "x") + + def test_bootstrap_empty_vaultprefixes_allowed_for_early_none_parse(self, getpass): + testfile_output = self.get_file_as_stdout( + os.path.join(self.testdir_v2, "values-secret-v2-bootstrap-empty-vaultprefixes.yaml") + ) + with self.assertRaises(AnsibleExitJson) as ansible_err: + set_module_args( + { + "values_secrets_plaintext": testfile_output, + "secrets_phase": "early", + } + ) + parse_secrets_info.main() + + ret = ansible_err.exception.args[0] + self.assertEqual(ret["failed"], False) + ps = ret["parsed_secrets"]["no-prefix-needed"] + self.assertEqual(ps["vault_prefixes"], []) + self.assertEqual(ps["fields"]["token"], "abc") + + def test_bootstrap_dup_name_late_parses_unique_secrets_only(self, getpass): + testfile_output = self.get_file_as_stdout( + os.path.join(self.testdir_v2, "values-secret-v2-bootstrap-dup-mixed-late.yaml") + ) + with self.assertRaises(AnsibleExitJson) as ansible_err: + set_module_args( + { + "values_secrets_plaintext": testfile_output, + "secrets_phase": "late", + } + ) + parse_secrets_info.main() + + ret = ansible_err.exception.args[0] + self.assertEqual(ret["failed"], False) + self.assertEqual(set(ret["parsed_secrets"].keys()), {"late-unique"}) + self.assertEqual( + ret["parsed_secrets"]["late-unique"]["fields"]["only_late"], "lateval" + ) + + def test_bootstrap_dup_name_default_late_omits_secrets_when_name_collides(self, getpass): + """Default secrets_phase is late; same-name secrets entries are still omitted.""" + testfile_output = self.get_file_as_stdout( + os.path.join(self.testdir_v2, "values-secret-v2-bootstrap-dup-across.yaml") + ) + with self.assertRaises(AnsibleExitJson) as ansible_err: + set_module_args( + { + "values_secrets_plaintext": testfile_output, + } + ) + parse_secrets_info.main() + + ret = ansible_err.exception.args[0] + self.assertEqual(ret["failed"], False) + self.assertEqual(len(ret["parsed_secrets"]), 0) + + def test_duplicate_secret_names_within_bootstrap_section_fails(self, getpass): + yaml_content = """ +version: "2.0" +backingStore: vault +bootstrap_secrets: + - name: dup + targetNamespaces: + - default + vaultPrefixes: + - hub + fields: + - name: f1 + value: a + onMissingValue: error + - name: dup + targetNamespaces: + - default + vaultPrefixes: + - hub + fields: + - name: f2 + value: b + onMissingValue: error +secrets: [] +""" + with self.assertRaises(AnsibleFailJson) as ansible_err: + set_module_args( + { + "values_secrets_plaintext": yaml_content, + "secrets_phase": "early", + } + ) + parse_secrets_info.main() + + ret = ansible_err.exception.args[0] + self.assertEqual(ret["failed"], True) + assert "duplicate secret names in bootstrap_secrets" in ret["args"][1] + assert "dup" in ret["args"][1] + + def test_invalid_secrets_phase_fails(self, getpass): + testfile_output = self.get_file_as_stdout( + os.path.join(self.testdir_v2, "values-secret-v2-base.yaml") + ) + with self.assertRaises(AnsibleFailJson) as ansible_err: + set_module_args( + { + "values_secrets_plaintext": testfile_output, + "secrets_phase": "midnight", + } + ) + parse_secrets_info.main() + + ret = ansible_err.exception.args[0] + self.assertEqual(ret["failed"], True) + # Invalid choice fails in AnsibleModule before run(); message is in msg, not args[1]. + msg = ret.get("msg") or ( + ret["args"][1] if len(ret.get("args", ())) > 1 else "" + ) + self.assertIn("early", msg) + self.assertIn("late", msg) + self.assertTrue( + "secrets_phase must be 'early' or 'late'" in msg + or "midnight" in msg.lower(), + msg, + ) + if __name__ == "__main__": unittest.main() diff --git a/tests/unit/test_vault_load_secrets_v2.py b/tests/unit/test_vault_load_secrets_v2.py index 0cc3f40..bb97b23 100644 --- a/tests/unit/test_vault_load_secrets_v2.py +++ b/tests/unit/test_vault_load_secrets_v2.py @@ -450,8 +450,87 @@ def test_ensure_error_secrets_same_name(self, getpass): ret = ansible_err.exception.args[0] self.assertEqual(ret["failed"], True) assert ( - ret["args"][1] == "You cannot have duplicate secret names: ['config-demo']" + ret["args"][1] + == "You cannot have duplicate secret names in secrets: ['config-demo']" + ) + + def test_vault_load_skips_secrets_when_secret_name_duplicates_bootstrap(self, getpass): + """vault_load_secrets does not inject bootstrap or duplicate-name secrets[] entries.""" + set_module_args( + { + "values_secrets": os.path.join( + self.testdir_v2, "values-secret-v2-bootstrap-dup-across.yaml" + ), + } + ) + with patch.object( + load_secrets_v2.LoadSecretsV2, "_run_command" + ) as mock_run_command: + mock_run_command.return_value = (0, "ok", "") + with self.assertRaises(AnsibleExitJson): + vault_load_secrets.main() + cmds = [c[0][0] for c in mock_run_command.call_args_list if c[0]] + blob = "\n".join(cmds) + self.assertNotIn("a='x'", blob) + self.assertNotIn("b='y'", blob) + + def test_vault_load_mixed_secrets_injects_only_non_bootstrap_vault_entries(self, getpass): + set_module_args( + { + "values_secrets": os.path.join( + self.testdir_v2, "values-secret-v2-bootstrap-dup-mixed-late.yaml" + ), + } ) + with patch.object( + load_secrets_v2.LoadSecretsV2, "_run_command" + ) as mock_run_command: + mock_run_command.return_value = (0, "ok", "") + with self.assertRaises(AnsibleExitJson) as result: + vault_load_secrets.main() + self.assertTrue(result.exception.args[0]["changed"]) + cmds = [c[0][0] for c in mock_run_command.call_args_list if c[0]] + blob = "\n".join(cmds) + self.assertNotIn("from_bootstrap='bootval'", blob) + self.assertNotIn("from_secrets_section=", blob) + self.assertIn("only_late='lateval'", blob) + + def test_vault_load_fails_duplicate_secret_names_within_bootstrap(self, getpass): + with self.assertRaises(AnsibleFailJson) as ansible_err: + set_module_args( + { + "values_secrets": os.path.join( + self.testdir_v2, "values-secret-v2-bootstrap-dup-within.yaml" + ), + } + ) + vault_load_secrets.main() + + ret = ansible_err.exception.args[0] + self.assertEqual(ret["failed"], True) + assert "duplicate secret names in bootstrap_secrets" in ret["args"][1] + + def test_vault_load_bootstrap_only_does_not_kv_inject_secret_fields(self, getpass): + """bootstrap_secrets are never written by vault_load_secrets (early K8s / none only).""" + set_module_args( + { + "values_secrets": os.path.join( + self.testdir_v2, "values-secret-v2-bootstrap-empty-vaultprefixes.yaml" + ), + } + ) + with patch.object( + load_secrets_v2.LoadSecretsV2, "_run_command" + ) as mock_run_command: + mock_run_command.return_value = (0, "ok", "") + with self.assertRaises(AnsibleExitJson) as result: + vault_load_secrets.main() + cmds = [c[0][0] for c in mock_run_command.call_args_list if c[0]] + blob = "\n".join(cmds) + self.assertNotIn("no-prefix-needed", blob) + self.assertNotIn("token='abc'", blob) + self.assertNotIn("vault kv put", blob) + self.assertNotIn("vault kv patch", blob) def test_ensure_error_fields_same_name(self, getpass): with self.assertRaises(AnsibleFailJson) as ansible_err: diff --git a/tests/unit/v2/values-secret-v2-bootstrap-and-late.yaml b/tests/unit/v2/values-secret-v2-bootstrap-and-late.yaml new file mode 100644 index 0000000..da5f2af --- /dev/null +++ b/tests/unit/v2/values-secret-v2-bootstrap-and-late.yaml @@ -0,0 +1,28 @@ +version: "2.0" + +backingStore: vault + +vaultPolicies: + basicPolicy: | + length=10 + rule "charset" { charset = "abcdefghijklmnopqrstuvwxyz" min-chars = 1 } + +bootstrap_secrets: + - name: bootstrap-only + targetNamespaces: + - default + vaultPrefixes: + - hub + fields: + - name: bootval + value: bootsecret + onMissingValue: error + +secrets: + - name: late-only + vaultPrefixes: + - hub + fields: + - name: lateval + value: latesecret + onMissingValue: error diff --git a/tests/unit/v2/values-secret-v2-bootstrap-dup-across.yaml b/tests/unit/v2/values-secret-v2-bootstrap-dup-across.yaml new file mode 100644 index 0000000..2a46ffb --- /dev/null +++ b/tests/unit/v2/values-secret-v2-bootstrap-dup-across.yaml @@ -0,0 +1,22 @@ +version: "2.0" +backingStore: vault + +bootstrap_secrets: + - name: config-demo + targetNamespaces: + - default + vaultPrefixes: + - hub + fields: + - name: a + value: x + onMissingValue: error + +secrets: + - name: config-demo + vaultPrefixes: + - hub + fields: + - name: b + value: y + onMissingValue: error diff --git a/tests/unit/v2/values-secret-v2-bootstrap-dup-mixed-late.yaml b/tests/unit/v2/values-secret-v2-bootstrap-dup-mixed-late.yaml new file mode 100644 index 0000000..35e2cd2 --- /dev/null +++ b/tests/unit/v2/values-secret-v2-bootstrap-dup-mixed-late.yaml @@ -0,0 +1,29 @@ +version: "2.0" +backingStore: vault + +bootstrap_secrets: + - name: shared-name + targetNamespaces: + - default + vaultPrefixes: + - hub + fields: + - name: from_bootstrap + value: bootval + onMissingValue: error + +secrets: + - name: shared-name + vaultPrefixes: + - hub + fields: + - name: from_secrets_section + value: should_not_apply_in_late_parse + onMissingValue: error + - name: late-unique + vaultPrefixes: + - hub + fields: + - name: only_late + value: lateval + onMissingValue: error diff --git a/tests/unit/v2/values-secret-v2-bootstrap-dup-within.yaml b/tests/unit/v2/values-secret-v2-bootstrap-dup-within.yaml new file mode 100644 index 0000000..55babf2 --- /dev/null +++ b/tests/unit/v2/values-secret-v2-bootstrap-dup-within.yaml @@ -0,0 +1,24 @@ +version: "2.0" +backingStore: vault + +bootstrap_secrets: + - name: dup + targetNamespaces: + - default + vaultPrefixes: + - hub + fields: + - name: f1 + value: a + onMissingValue: error + - name: dup + targetNamespaces: + - default + vaultPrefixes: + - hub + fields: + - name: f2 + value: b + onMissingValue: error + +secrets: [] diff --git a/tests/unit/v2/values-secret-v2-bootstrap-empty-vaultprefixes.yaml b/tests/unit/v2/values-secret-v2-bootstrap-empty-vaultprefixes.yaml new file mode 100644 index 0000000..db36015 --- /dev/null +++ b/tests/unit/v2/values-secret-v2-bootstrap-empty-vaultprefixes.yaml @@ -0,0 +1,14 @@ +version: "2.0" +backingStore: vault + +bootstrap_secrets: + - name: no-prefix-needed + targetNamespaces: + - apps + vaultPrefixes: [] + fields: + - name: token + value: abc + onMissingValue: error + +secrets: [] diff --git a/tests/unit/v2/values-secret-v2-bootstrap-missing-targetns.yaml b/tests/unit/v2/values-secret-v2-bootstrap-missing-targetns.yaml new file mode 100644 index 0000000..d3a352c --- /dev/null +++ b/tests/unit/v2/values-secret-v2-bootstrap-missing-targetns.yaml @@ -0,0 +1,20 @@ +version: "2.0" +backingStore: vault + +bootstrap_secrets: + - name: boot-without-ns + vaultPrefixes: + - hub + fields: + - name: k + value: v + onMissingValue: error + +secrets: + - name: late-ok + vaultPrefixes: + - hub + fields: + - name: k2 + value: v2 + onMissingValue: error diff --git a/tests/unit/v2/values-secret-v2-bootstrap-only.yaml b/tests/unit/v2/values-secret-v2-bootstrap-only.yaml new file mode 100644 index 0000000..dc6cb94 --- /dev/null +++ b/tests/unit/v2/values-secret-v2-bootstrap-only.yaml @@ -0,0 +1,13 @@ +version: "2.0" +backingStore: vault + +bootstrap_secrets: + - name: only-bootstrap + targetNamespaces: + - validated-patterns-secrets + vaultPrefixes: + - hub + fields: + - name: k + value: v + onMissingValue: error