Introduce Bootstrap Secrets

.github/workflows/ansible-lint.yml

@@ -14,7 +14,21 @@ jobs:
         with:
           persist-credentials: false
 
-      - name: Lint Ansible Playbook
-        uses: ansible/ansible-lint@5fac056c45595896c973fbde871f01f6cb14d74c
+      # The ansible-lint GitHub action installs via uv with a lock that tracks the newest
+      # ansible-core; ansible-galaxy collection install then hits Galaxy KeyError: 'results'
+      # on some 2.19+/2.20 clients. Pin ansible-core to the 2.18 series for a stable Galaxy client.
+      - name: Set up Python
+        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
         with:
-          setup_python: "true"
+          python-version: "3.12"
+
+      - name: Install ansible-lint and Galaxy-compatible ansible-core
+        run: |
+          python -m pip install --upgrade pip
+          pip install ansible-lint "ansible-core>=2.18.0,<2.19.0"
+
+      - name: Install Ansible Galaxy collection dependencies
+        run: ansible-galaxy collection install -r requirements.yml
+
+      - name: Run ansible-lint
+        run: ansible-lint roles playbooks plugins

.github/workflows/jsonschema.yaml

@@ -32,4 +32,4 @@ jobs:
       - name: Verify secrets json schema
         run: |
           set -e
-          for i in values-secret-v2-base values-secret-v2-generic-onlygenerate values-secret-v2-block-yamlstring; do echo "$i"; check-jsonschema --fill-defaults --schemafile ./roles/vault_utils/values-secrets.v2.schema.json "tests/unit/v2/$i.yaml"; done
+          for i in values-secret-v2-base values-secret-v2-generic-onlygenerate values-secret-v2-block-yamlstring values-secret-v2-bootstrap-and-late values-secret-v2-bootstrap-only; do echo "$i"; check-jsonschema --fill-defaults --schemafile ./roles/vault_utils/values-secrets.v2.schema.json "tests/unit/v2/$i.yaml"; done

.gitignore

@@ -13,3 +13,4 @@ super-linter.log
 golang-external-secrets/Chart.lock
 hashicorp-vault/Chart.lock
 tests/output
+ansible_collections/

Makefile

@@ -48,4 +48,4 @@ test: ansible-sanitytest ansible-unittest
 .PHONY: check-jsonschema
 check-jsonschema: ## Runs check-jsonschema against all unit test files except known broken ones
 	set -e; \
-	for i in values-secret-v2-base values-secret-v2-generic-onlygenerate values-secret-v2-block-yamlstring; do echo "$$i"; check-jsonschema --schemafile ./roles/vault_utils/values-secrets.v2.schema.json "tests/unit/v2/$$i.yaml"; done
+	for i in values-secret-v2-base values-secret-v2-generic-onlygenerate values-secret-v2-block-yamlstring values-secret-v2-bootstrap-and-late values-secret-v2-bootstrap-only; do echo "$$i"; check-jsonschema --schemafile ./roles/vault_utils/values-secrets.v2.schema.json "tests/unit/v2/$$i.yaml"; done

README.md

@@ -8,3 +8,14 @@ The main purpose of this collections are to:
 loading local secrets files into VP secrets stores.
 
 2. Help manage imperative and other utility functions of the cluster
+
+## Pattern repository directory (`pattern_dir`)
+
+Playbooks need the path to your pattern Git checkout (where **`values-global.yaml`**
+and related files live). Resolution order: extra var **`pattern_dir`**, environment
+variable **`PATTERN_DIR`**, then **`PWD`** and **`pwd`**.
+
+When running from the imperative container or another fixed working directory,
+pass the repository root explicitly, for example **`-e pattern_dir=/git/repo`** (or add
+equivalent extra vars via **`clusterGroup.imperative.extraPlaybookArgs`** in the
+clustergroup chart).

playbooks/determine_pattern_dir.yml

@@ -1,17 +1,20 @@
 ---
+# Same resolution as roles/pattern_settings/tasks/resolve_overrides.yml and README.md:
+# extra var pattern_dir, then env PATTERN_DIR, then env PWD, then output of pwd (trimmed).
 - name: Determine pattern dir
   hosts: localhost
   connection: local
   gather_facts: false
   become: false
-  vars:
-    pattern_dir: ''
   tasks:
-    - name: Fail if directory is not set
-      ansible.builtin.fail:
-        msg: "pattern_dir variable must be set"
-      when: pattern_dir | length == 0
+    - name: Resolve pattern_dir (same as pattern_settings)
+      ansible.builtin.include_role:
+        name: pattern_settings
+        tasks_from: resolve_overrides
 
-    - name: Set pattern_dir fact for future plays
-      ansible.builtin.set_fact:
-        pattern_dir: '{{ pattern_dir }}'
+    - name: Fail if pattern_dir is still empty after resolution
+      ansible.builtin.fail:
+        msg: >-
+          Could not resolve pattern_dir. Set -e pattern_dir=/path/to/pattern, export PATTERN_DIR,
+          or run from your pattern repository so PWD / pwd points at the directory that contains values-global.yaml.
+      when: pattern_dir | default('', true) | string | trim | length == 0

playbooks/determine_secretstore_backend.yml

@@ -7,8 +7,6 @@
   connection: local
   gather_facts: false
   become: false
-  vars:
-    pattern_dir: "."
   tasks:
     - name: Set fact for secretStore backend
       ansible.builtin.set_fact:

playbooks/display_secrets_info.yml

@@ -23,19 +23,54 @@
       ansible.builtin.include_role:
         name: find_vp_secrets
 
+    # find_vp_secrets may set values_secrets_data as a dict (from_yaml) or as raw text; parse_secrets_info needs a YAML string.
     # find_vp_secrets will return a plaintext data structure called values_secrets_data
     # This will allow us to determine schema version and which backend to use
     - name: Determine how to load secrets
       ansible.builtin.set_fact:
         secrets_yaml: "{{ values_secrets_data if values_secrets_data is not string else values_secrets_data | from_yaml }}"
 
-    - name: Parse secrets data
+    - name: Normalize values-secret content for parse_secrets_info
+      ansible.builtin.set_fact:
+        values_secrets_plaintext_for_parse: >-
+          {{ values_secrets_data if values_secrets_data is string else (values_secrets_data | to_yaml) }}
+
+    - name: Resolve secrets backing store for parse module
+      ansible.builtin.set_fact:
+        secrets_backing_store_for_parse: >-
+          {{ secrets_backing_store
+             | default(values_global.global.secretStore.backend | default('vault'), true) }}
+
+    - name: Parse early-phase (bootstrap) secrets
       no_log: '{{ hide_sensitive_output }}'
       parse_secrets_info:
-        values_secrets_plaintext: "{{ values_secrets_data }}"
-        secrets_backing_store: "{{ secrets_backing_store }}"
-      register: secrets_results
+        values_secrets_plaintext: "{{ values_secrets_plaintext_for_parse }}"
+        secrets_backing_store: "{{ secrets_backing_store_for_parse }}"
+        secrets_phase: early
+      register: secrets_results_early
+
+    - name: Parse late-phase secrets
+      no_log: '{{ hide_sensitive_output }}'
+      parse_secrets_info:
+        values_secrets_plaintext: "{{ values_secrets_plaintext_for_parse }}"
+        secrets_backing_store: "{{ secrets_backing_store_for_parse }}"
+        secrets_phase: late
+      register: secrets_results_late
+
+    - name: Assemble secrets display (early and late phases)
+      ansible.builtin.set_fact:
+        secrets_display_by_phase:
+          vault_policies: "{{ secrets_results_late.vault_policies | default(secrets_results_early.vault_policies | default({})) }}"
+          secret_store_namespace: "{{ secrets_results_late.secret_store_namespace | default('validated-patterns-secrets') }}"
+          early_phase:
+            parsed_secrets: "{{ secrets_results_early.parsed_secrets | default({}) }}"
+            kubernetes_secret_objects: "{{ secrets_results_early.kubernetes_secret_objects | default([]) }}"
+            unique_vault_prefixes: "{{ secrets_results_early.unique_vault_prefixes | default([]) }}"
+          late_phase:
+            parsed_secrets: "{{ secrets_results_late.parsed_secrets | default({}) }}"
+            kubernetes_secret_objects: "{{ secrets_results_late.kubernetes_secret_objects | default([]) }}"
+            unique_vault_prefixes: "{{ secrets_results_late.unique_vault_prefixes | default([]) }}"
 
-    - name: Display secrets data
+    - name: Display secrets data (early and late phases)
       ansible.builtin.debug:
-        var: secrets_results
+        var: secrets_display_by_phase

playbooks/install.yml

@@ -1,8 +1,11 @@
 ---
+- name: Load bootstrap secrets early phase
+  ansible.builtin.import_playbook: load_bootstrap_secrets.yml
+
 - name: Install the pattern via pattern-install chart
   ansible.builtin.import_playbook: operator_deploy.yml
 
-- name: Load secrets (if not explicitly disabled in values-global.yaml)
+- name: Load secrets late phase (if not explicitly disabled in values-global.yaml)
   ansible.builtin.import_playbook: load_secrets.yml
 
 - name: Wait for pattern to finish installation (all Argo apps should be healthy/synced)

playbooks/load_bootstrap_secrets.yml

@@ -0,0 +1,19 @@
+---
+# Loads only bootstrap_secrets (early phase), before pattern operator install.
+# Always uses the Kubernetes / none injector (never vault_utils), even when values-global uses vault.
+# Vault is not available yet at this stage; bootstrap material is never written to Vault from here.
+# Runs even when .global.secretLoader.disabled is true (late-phase loading still honors that flag).
+# Use playbooks/load_secrets.yml for late-phase secrets.
+- name: Decide whether to load bootstrap secrets
+  hosts: localhost
+  connection: local
+  gather_facts: false
+  roles:
+    - role: pattern_settings
+
+  tasks:
+    - name: Load bootstrap secrets early phase
+      ansible.builtin.include_role:
+        name: load_secrets
+      vars:
+        secrets_phase: early

playbooks/load_secrets.yml

@@ -1,5 +1,9 @@
 ---
-- name: Decide whether to load secrets
+# Loads late-phase `secrets` entries. If `bootstrap_secrets` were not applied earlier in
+# the same playbook on this host (see `bootstrap_secrets_loaded_this_run`), the
+# load_secrets role applies them once before late parse/inject. Use load_bootstrap_secrets.yml
+# for explicit early-only bootstrap loading (e.g. before operator install).
+- name: Decide whether to load late-phase secrets
   hosts: localhost
   connection: local
   gather_facts: false
@@ -11,13 +15,14 @@
       ansible.builtin.set_fact:
         secret_loader_disabled: "{{ values_global.global.secretLoader.disabled | default(false) | bool }}"
 
-    - name: Load secrets (when enabled)
+    - name: Load late-phase secrets (when enabled)
       ansible.builtin.include_role:
         name: load_secrets
       when: not secret_loader_disabled
 
-    - name: Print secret loading disabled message
+    - name: Print late-phase secret loading disabled message
       ansible.builtin.debug:
         msg: |
-          Secrets loading is currently disabled. To enable, update the value of '.global.secretLoader.disabled' in 'values-global.yaml' to 'false'.
+          Late-phase secrets loading is currently disabled. Bootstrap secrets were still applied when present.
+          To enable late-phase loading, update the value of '.global.secretLoader.disabled' in 'values-global.yaml' to 'false'.
       when: secret_loader_disabled

playbooks/process_secrets.yml

@@ -9,6 +9,7 @@
     pattern_dir: '.'
     secrets_backing_store: 'vault'
     tasks_from: 'push_parsed_secrets'
+    secrets_phase: late
   tasks:
     - name: "Run secret-loading pre-requisites"
       ansible.builtin.include_role:
@@ -23,21 +24,49 @@
       ansible.builtin.set_fact:
         secrets_yaml: "{{ values_secrets_data if values_secrets_data is not string else values_secrets_data | from_yaml }}"
 
+    - name: Ensure bootstrap secrets before late phase when not already applied this run
+      ansible.builtin.include_role:
+        name: load_secrets
+        tasks_from: ensure_bootstrap_before_late.yml
+      when:
+        - secrets_phase | default('late') == 'late'
+        - not (bootstrap_secrets_loaded_this_run | default(false))
+        - (secrets_yaml.version | default('2.0')) is version('2.0', '>=')
+        - (secrets_yaml.bootstrap_secrets | default([]) | length) > 0
+
     - name: Parse secrets data
       no_log: '{{ hide_sensitive_output | default(true) }}'
       parse_secrets_info:
         values_secrets_plaintext: "{{ values_secrets_data }}"
         secrets_backing_store: "{{ secrets_backing_store }}"
+        secrets_phase: "{{ secrets_phase }}"
       register: secrets_results
 
-    # Use the k8s secrets loader when explicitly requested
-    - name: Determine role to use to load secrets
+    - name: Assert values-secret v2 when loading bootstrap (early) secrets
+      ansible.builtin.assert:
+        that:
+          - (secrets_yaml.version | default('1.0')) is version('2.0', '>=')
+        fail_msg: >-
+          Bootstrap secret loading (secrets_phase=early) requires values-secret format version 2.0 or newer.
+        success_msg: values-secret version is 2.0 or newer; bootstrap (early) loading is allowed.
+      when: secrets_phase | default('late') == 'early'
+
+    - name: Select Kubernetes secrets loader for bootstrap (early) phase
+      ansible.builtin.set_fact:
+        secrets_role: 'k8s_secret_utils'
+        tasks_from: 'inject_k8s_secrets'
+      when:
+        - secrets_phase | default('late') == 'early'
+        - (secrets_yaml.version | default('2.0')) is version('2.0', '>=')
+
+    - name: Select Kubernetes secrets loader for late phase (kubernetes or none backend)
       ansible.builtin.set_fact:
         secrets_role: 'k8s_secret_utils'
         tasks_from: 'inject_k8s_secrets'
       when:
+        - secrets_phase | default('late') != 'early'
         - secrets_backing_store == "kubernetes" or secrets_backing_store == "none"
-        - secrets_yaml['version'] | default('2.0') >= '2.0'
+        - (secrets_yaml.version | default('2.0')) is version('2.0', '>=')
 
     # secrets_role will have been changed from the default if needed
     - name: Load secrets using designated role and tasks
@@ -49,3 +78,4 @@
         vault_policies: "{{ secrets_results['vault_policies'] }}"
         parsed_secrets: "{{ secrets_results['parsed_secrets'] }}"
         unique_vault_prefixes: "{{ secrets_results['unique_vault_prefixes'] | default([]) }}"
+        secrets_install_phase: "{{ secrets_phase | default('late') }}"

plugins/module_utils/load_secrets_v2.py

@@ -14,7 +14,10 @@
 # under the License.
 
 """
-Module that implements V2 of the values-secret.yaml spec
+Module that implements V2 of the values-secret.yaml spec.
+
+vault_load_secrets writes only the top-level ``secrets`` list to Vault; ``bootstrap_secrets`` are
+validated but never pushed to Vault (bootstrap targets Kubernetes via the early none injector).
 """
 from __future__ import absolute_import, division, print_function
 
@@ -75,41 +78,52 @@ def _get_backingstore(self):
         """
         return str(self.syaml.get("backingStore", "vault"))
 
-    def _get_secrets(self):
-        return self.syaml.get("secrets", {})
+    def _get_bootstrap_secrets(self):
+        bootstrap = self.syaml.get("bootstrap_secrets", [])
+        if bootstrap is None or bootstrap == "None":
+            return []
+        return bootstrap
 
     def _validate_secrets(self):
         secrets = self._get_secrets()
-        if len(secrets) == 0:
+        bootstrap_secrets = self._get_bootstrap_secrets()
+        if len(secrets) == 0 and len(bootstrap_secrets) == 0:
             self.module.fail_json("No secrets found")
 
-        # Validate each secret and collect names for duplicate checking
+        bootstrap_names = []
+        for secret in bootstrap_secrets:
+            result = self._validate_secret(secret, is_bootstrap=True)
+            if not result[0]:
+                return result
+            bootstrap_names.append(secret["name"])
+
         secret_names = []
         for secret in secrets:
-            result = self._validate_secret(secret)
+            result = self._validate_secret(secret, is_bootstrap=False)
             if not result[0]:
                 return result
             secret_names.append(secret["name"])
 
-        # Check for duplicate secret names
-        dupes = find_dupes(secret_names)
-        if len(dupes) > 0:
-            return (False, f"You cannot have duplicate secret names: {dupes}")
+        dupes_bootstrap = find_dupes(bootstrap_names)
+        if len(dupes_bootstrap) > 0:
+            return (False, f"You cannot have duplicate secret names in bootstrap_secrets: {dupes_bootstrap}")
+        dupes_secrets = find_dupes(secret_names)
+        if len(dupes_secrets) > 0:
+            return (False, f"You cannot have duplicate secret names in secrets: {dupes_secrets}")
 
         return (True, "")
 
-    def _validate_secret(self, secret):
+    def _validate_secret(self, secret, is_bootstrap=False):
         """Validate a single secret configuration"""
         # Check mandatory fields
         if "name" not in secret:
             return (False, f"Secret {secret} is missing name")
 
-        secret_name = secret["name"]
-
-        # Validate vault prefixes
-        result = self._validate_vault_prefixes(secret)
-        if not result[0]:
-            return result
+        # Bootstrap secrets are only applied via early K8s (none) load in phased installs; vaultPrefixes are not used.
+        if not is_bootstrap:
+            result = self._validate_vault_prefixes(secret)
+            if not result[0]:
+                return result
 
         # Validate fields
         result = self._validate_secret_fields(secret)
@@ -298,10 +312,26 @@ def inject_secrets(self):
         # This must come first as some passwords might depend on vault policies to exist.
         # It is a noop when no policies are defined
         self.inject_vault_policies()
+        bootstrap_secrets = self._get_bootstrap_secrets()
         secrets = self._get_secrets()
+        bootstrap_names = {s.get("name") for s in bootstrap_secrets if s.get("name")}
+        secrets_filtered = [s for s in secrets if s.get("name") not in bootstrap_names]
+        skipped = {s.get("name") for s in secrets if s.get("name") in bootstrap_names}
+        for n in sorted(skipped):
+            self.module.warn(
+                "Omitting secrets entry %r because it duplicates a bootstrap_secrets name "
+                "(bootstrap material is not written to Vault from this module)." % (n,)
+            )
+
+        if bootstrap_secrets:
+            self.module.warn(
+                "bootstrap_secrets are validated but not written to Vault; Vault is not used "
+                "for bootstrap material. Apply bootstrap secrets with the early-phase Kubernetes "
+                "(none) injector (for example load_bootstrap_secrets)."
+            )
 
-        total_secrets = 0  # Counter for all the secrets uploaded
-        for s in secrets:
+        total_secrets = 0  # Counter for all the secrets uploaded (secrets[] only; never bootstrap_secrets)
+        for s in secrets_filtered:
             counter = 0  # This counter is to use kv put on first secret and kv patch on latter
             sname = s.get("name")
             fields = s.get("fields", [])