diff --git a/docs/dictionary/en-custom.txt b/docs/dictionary/en-custom.txt index de4efbe7c..999f9d1a4 100644 --- a/docs/dictionary/en-custom.txt +++ b/docs/dictionary/en-custom.txt @@ -1,3 +1,4 @@ +ACL AES APIs Amartya @@ -16,6 +17,7 @@ LDAP LLM MachineConfig Marjanovic +MiB Nemanja NICs NodeHealthCheck diff --git a/roles/devscripts/README.md b/roles/devscripts/README.md index c4e4a22eb..ab2d977b2 100644 --- a/roles/devscripts/README.md +++ b/roles/devscripts/README.md @@ -59,6 +59,7 @@ networks. * `cifmw_devscripts_cpu_passthrough` (bool) Enable host-passthrough cpu model to virtual machine instead of using QEMU on nested environment. Defaults to `false`. +* `cifmw_devscripts_sno` (bool) Enable Single Node OpenShift. Defaults to `false`. ### Secrets management diff --git a/roles/devscripts/defaults/main.yml b/roles/devscripts/defaults/main.yml index 3247581e4..3c0c63917 100644 --- a/roles/devscripts/defaults/main.yml +++ b/roles/devscripts/defaults/main.yml @@ -75,3 +75,4 @@ cifmw_devscripts_installer_timeout: 7200 # 2 hours cifmw_devscripts_etcd_slow_profile: true cifmw_devscripts_disable_console: false cifmw_devscripts_cpu_passthrough: false +cifmw_devscripts_sno: false diff --git a/roles/devscripts/templates/conf_ciuser.j2 b/roles/devscripts/templates/conf_ciuser.j2 index f143b68dc..942b1a9b0 100644 --- a/roles/devscripts/templates/conf_ciuser.j2 +++ b/roles/devscripts/templates/conf_ciuser.j2 @@ -7,6 +7,22 @@ set +x export CI_TOKEN=$(cat {{ cifmw_devscripts_repo_dir }}/ci_token) set -x + +{% if cifmw_devscripts_sno %} +# SNO configuration +# https://github.com/openshift-metal3/dev-scripts/blob/master/AGENTS.md +{% for item in ['working_dir', 'assets_extra_folder', 'openshift_release_type', + 'openshift_version', 'cluster_name', 'base_domain', 'ntp_servers', + 'external_subnet_v4', 'ip_stack', 'agent_e2e_test_scenario', + 'agent_platform_type'] %} +export {{ item.upper() }}="{{ cifmw_devscripts_config[item] }}" +{% endfor %} +export NUM_MASTERS=1 +export NUM_WORKERS=0 +{% else %} + {% for item in cifmw_devscripts_config %} export {{ item.upper() }}="{{ cifmw_devscripts_config[item] }}" {% endfor %} + +{% endif %} diff --git a/roles/devscripts_sno/README.md b/roles/devscripts_sno/README.md new file mode 100644 index 000000000..d682368f5 --- /dev/null +++ b/roles/devscripts_sno/README.md @@ -0,0 +1,177 @@ +# devscripts_sno + +Deploy a Single Node OpenShift (SNO) cluster on libvirt/KVM using the +[dev-scripts](https://github.com/openshift-metal3/dev-scripts) agent installer +(`make agent`). + +This role is a thin wrapper around the `devscripts` role: it prepares secrets +and host prerequisites, runs the dev-scripts configuration and host-prep steps, +executes `make agent`, and copies the resulting kubeconfig to `~/.kube/config`. + +It complements the full `devscripts` deployment path (baremetal/libvirt with +`make all`) and the physical bare metal path documented in +[bm_sno](../bm_sno/README.md) (`cifmw_bm_sno: true`). + +When used from the reproducer, set `cifmw_devscripts_sno: true` so the +`devscripts` configuration template exports SNO-specific variables (see +[dev-scripts AGENTS.md](https://github.com/openshift-metal3/dev-scripts/blob/master/AGENTS.md)). +That flag is separate from including this role by name; reproducer scenarios +such as `scenarios/reproducers/va-hci-minimal-sno.yml` combine the flag with +`cifmw_devscripts_config_overrides` for a one-master layout. + +## Parameters + +### Role-specific + +| Parameter | Type | Default | Description | +| --- | --- | --- | --- | +| `cifmw_devscripts_sno_installer_timeout` | int | `7200` | Seconds before `make agent` is killed by `timeout` (2 hours) | +| `cifmw_devscripts_sno_repo_dir` | str | `~/src/github.com/openshift-metal3/dev-scripts` | Path to the dev-scripts repository | +| `cifmw_devscripts_sno_data_dir` | str | `~/ci-framework-data` | Base directory for CI Framework data on the host | +| `cifmw_devscripts_sno_artifacts_dir` | str | `{{ cifmw_devscripts_sno_data_dir }}/artifacts` | Directory for role script artifacts and logs | + +### Required for standalone use + +When running this role outside the reproducer playbook, provide secret content +(the role writes files from these variables in `pre.yml`): + +| Parameter | Description | +| --- | --- | +| `cifmw_manage_secrets_citoken_content` | CI token string written to `cifmw_manage_secrets_citoken_file` | +| `cifmw_manage_secrets_pullsecret_content` | Pull secret JSON written to `cifmw_manage_secrets_pullsecret_file` | + +Default secret file paths (from `defaults/main.yml`): + +| Parameter | Default | +| --- | --- | +| `cifmw_manage_secrets_citoken_file` | `{{ ansible_user_dir }}/secrets/ci_token` | +| `cifmw_manage_secrets_pullsecret_file` | `{{ ansible_user_dir }}/secrets/pull_secret.json` | + +Alternatively, use the file-based variables documented in the +[devscripts role README](../devscripts/README.md#secrets-management) if you +integrate with `manage_secrets` directly. + +### SNO and dev-scripts configuration + +Set `cifmw_devscripts_sno: true` so `conf_ciuser.j2` exports the reduced SNO +variable set (`NUM_MASTERS=1`, `NUM_WORKERS=0`, and agent-related keys). + +Cluster version, networking, and VM sizing are controlled through +`cifmw_devscripts_config_overrides` (and optional +`cifmw_devscripts_config_overrides_patch.*` keys), same as the `devscripts` +role. See [devscripts README](../devscripts/README.md#parameters) and +[Supported keys in cifmw_devscripts_config_overrides](../devscripts/README.md#supported-keys-in-cifmw_devscripts_config_overrides). + +Common SNO overrides (also used in `scenarios/reproducers/va-hci-minimal-sno.yml`): + +| Key | Example | Description | +| --- | --- | --- | +| `num_masters` | `1` | Single control-plane node | +| `master_memory` | `16384` | Memory (MiB) for the master VM | +| `master_disk` | `120` | Root disk size (GiB) | +| `master_vcpu` | `12` | vCPUs for the master VM | +| `agent_e2e_test_scenario` | `SNO_IPV4` | Agent installer test scenario | +| `agent_platform_type` | `none` | Agent platform type | +| `openshift_version` | `stable-4.18` | OpenShift version (minor or `stable-X.Y`) | +| `openshift_release_type` | `ga` | Release type (`nightly`, `ga`, `okd`) | +| `external_subnet_v4` | `192.168.111.0/24` | External network for the cluster | +| `ip_stack` | `v4` | IP stack (`v4`, `v6`, `v6v4`) | + +Additional dev-scripts variables (for example `cifmw_devscripts_cpu_passthrough`, +`cifmw_devscripts_host_bm_net_ip_addr`) follow the `devscripts` role +documentation. + +## Task files + +| Task file | Description | +| --- | --- | +| `main.yml` | Orchestrates pre, QEMU ACL, setup, and post phases | +| `pre.yml` | Maps role vars to `devscripts`, writes secrets, installs packages | +| `prepare_qemu_home_access.yml` | Installs `acl`/`qemu-kvm` and grants `qemu` traverse on the user home | +| `setup.yml` | Runs `devscripts` `build_config.yml` and `100_pre.yml`, then `make agent` | +| `post.yml` | Copies cluster kubeconfig to `~/.kube/config` | + +## Examples + +### Standalone playbook + +See `example-playbook.yaml` in this role directory: + +```bash +ansible-playbook \ + -e @secrets.yaml \ + -e @scenarios/reproducers/va-hci-minimal-sno.yml \ + -i inventory.yaml \ + roles/devscripts_sno/example-playbook.yaml +``` + +Minimal `secrets.yaml` for standalone runs: + +```yaml +cifmw_devscripts_sno: true +cifmw_manage_secrets_citoken_content: "{{ lookup('env', 'CI_TOKEN') }}" +cifmw_manage_secrets_pullsecret_content: | + {{ lookup('file', lookup('env', 'HOME') ~ '/pull-secret') }} + +cifmw_devscripts_config_overrides: + openshift_version: "stable-4.18" + openshift_release_type: ga + num_masters: 1 + master_memory: 16384 + master_disk: 120 + master_vcpu: 12 + external_subnet_v4: 192.168.111.0/24 + ip_stack: v4 + agent_e2e_test_scenario: SNO_IPV4 + agent_platform_type: none +``` + +### Reproducer scenario (libvirt SNO) + +From `scenarios/reproducers/va-hci-minimal-sno.yml`: + +```yaml +cifmw_devscripts_sno: true +cifmw_reproducer_allow_one_ocp: true +cifmw_devscripts_cpu_passthrough: true +cifmw_devscripts_host_bm_net_ip_addr: "192.168.111.1" + +cifmw_devscripts_config_overrides: + num_masters: 1 + master_memory: 16384 + master_disk: 120 + master_vcpu: 12 + external_subnet_v4: 192.168.111.0/24 + ip_stack: v4 + agent_e2e_test_scenario: SNO_IPV4 + agent_platform_type: none + openshift_release_type: ga + openshift_version: stable-4.18 +``` + +After deployment, credentials are under the dev-scripts auth directory: + +```bash +export KUBECONFIG=~/src/github.com/openshift-metal3/dev-scripts/ocp/ocp/auth/kubeconfig +oc login -u kubeadmin \ + -p "$(cat ~/src/github.com/openshift-metal3/dev-scripts/ocp/ocp/auth/kubeadmin-password)" +``` + +## SNO deployment methods in CI Framework + +| Flag | Role / method | Environment | +| --- | --- | --- | +| `cifmw_devscripts_sno: true` | dev-scripts agent (`make agent`) on libvirt/KVM | Virtual machine on the hypervisor | +| `cifmw_bm_sno: true` | `bm_sno` (agent-based, iDRAC Redfish) | Physical bare metal host | + +See [reproducer README](../reproducer/README.md#sno-deployment-methods) for how +these paths fit into full reproducer jobs. + +## References + +* [devscripts role](../devscripts/README.md) +* [bm_sno role](../bm_sno/README.md) +* [dev-scripts](https://github.com/openshift-metal3/dev-scripts) +* [dev-scripts agent installer](https://github.com/openshift-metal3/dev-scripts/tree/master/agent) +* [dev-scripts AGENTS.md](https://github.com/openshift-metal3/dev-scripts/blob/master/AGENTS.md) +* [dev-scripts config_example.sh](https://github.com/openshift-metal3/dev-scripts/blob/master/config_example.sh) diff --git a/roles/devscripts_sno/defaults/main.yml b/roles/devscripts_sno/defaults/main.yml new file mode 100644 index 000000000..4a74238c4 --- /dev/null +++ b/roles/devscripts_sno/defaults/main.yml @@ -0,0 +1,28 @@ +--- +# Copyright Red Hat, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + + +# All variables intended for modification should be placed in this file. +# All variables within this role should have a prefix of "cifmw_devscripts_sno" +# +cifmw_manage_secrets_citoken_file: "{{ ansible_user_dir }}/secrets/ci_token" +cifmw_manage_secrets_pullsecret_file: "{{ ansible_user_dir }}/secrets/pull_secret.json" +openshift_version: "4.18.41" + +cifmw_devscripts_sno_installer_timeout: 7200 +cifmw_devscripts_sno_repo_dir: "{{ ansible_user_dir }}/src/github.com/openshift-metal3/dev-scripts" +cifmw_devscripts_sno_data_dir: "{{ ansible_user_dir }}/ci-framework-data" +cifmw_devscripts_sno_artifacts_dir: "{{ cifmw_devscripts_sno_data_dir }}/artifacts" diff --git a/roles/devscripts_sno/example-playbook.yaml b/roles/devscripts_sno/example-playbook.yaml new file mode 100644 index 000000000..125a872f2 --- /dev/null +++ b/roles/devscripts_sno/example-playbook.yaml @@ -0,0 +1,13 @@ +--- +# Example usage: +# ansible-playbook \ +# -e @secrets.yaml -e @scenarios/reproducers/va-hci-minimal-sno.yml \ +# -i inventory.yaml \ +# example-playbook.yaml +- name: Deploying SNO host + hosts: sno.dev + gather_facts: true + tasks: + - name: Run devscript sno + ansible.builtin.include_role: + name: devscripts_sno diff --git a/roles/devscripts_sno/meta/main.yml b/roles/devscripts_sno/meta/main.yml new file mode 100644 index 000000000..62b983925 --- /dev/null +++ b/roles/devscripts_sno/meta/main.yml @@ -0,0 +1,28 @@ +--- +# Copyright Red Hat, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + + +galaxy_info: + author: CI Framework + description: CI Framework Role -- devscripts_sno + company: Red Hat + license: Apache-2.0 + min_ansible_version: "2.14" + namespace: cifmw + galaxy_tags: + - cifmw + +dependencies: [] diff --git a/roles/devscripts_sno/tasks/main.yml b/roles/devscripts_sno/tasks/main.yml new file mode 100644 index 000000000..44b49bf5b --- /dev/null +++ b/roles/devscripts_sno/tasks/main.yml @@ -0,0 +1,28 @@ +--- +# Copyright Red Hat, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + + +- name: Configure secrets and install required packages + ansible.builtin.include_tasks: pre.yml + +- name: Allow libvirt qemu user to access files under the user home directory + ansible.builtin.include_tasks: prepare_qemu_home_access.yml + +- name: Configure devscripts and run make agent command + ansible.builtin.include_tasks: setup.yml + +- name: Make post actions + ansible.builtin.include_tasks: post.yml diff --git a/roles/devscripts_sno/tasks/post.yml b/roles/devscripts_sno/tasks/post.yml new file mode 100644 index 000000000..c09410305 --- /dev/null +++ b/roles/devscripts_sno/tasks/post.yml @@ -0,0 +1,13 @@ +--- +- name: Create .kube directory + ansible.builtin.file: + path: "{{ ansible_user_dir }}/.kube" + state: directory + mode: "0750" + +- name: Copy devscripts kubeconfig to .kube + ansible.builtin.copy: + src: "{{ cifmw_devscripts_sno_repo_dir }}/ocp/ocp/auth/kubeconfig" + dest: "{{ ansible_user_dir }}/.kube/config" + remote_src: true + mode: "0640" diff --git a/roles/devscripts_sno/tasks/pre.yml b/roles/devscripts_sno/tasks/pre.yml new file mode 100644 index 000000000..7fa99b831 --- /dev/null +++ b/roles/devscripts_sno/tasks/pre.yml @@ -0,0 +1,35 @@ +--- +- name: Re-assign devscripts_sno vars with devscripts role vars + ansible.builtin.set_fact: + cifmw_devscripts_repo_dir: "{{ cifmw_devscripts_sno_repo_dir }}" + cifmw_devscripts_data_dir: "{{ cifmw_devscripts_sno_data_dir }}" + cifmw_devscripts_artifacts_dir: "{{ cifmw_devscripts_sno_artifacts_dir }}" + cifmw_devscripts_installer_timeout: "{{ cifmw_devscripts_sno_installer_timeout }}" + +- name: Ensure secret dir exists + ansible.builtin.file: + path: "{{ item | dirname }}" + state: directory + mode: "0755" + loop: + - "{{ cifmw_manage_secrets_citoken_file }}" + - "{{ cifmw_manage_secrets_pullsecret_file }}" + +- name: Copy required content - ci token + no_log: true + ansible.builtin.copy: + content: "{{ cifmw_manage_secrets_citoken_content }}" + dest: "{{ cifmw_manage_secrets_citoken_file }}" + mode: "0640" + +- name: Copy required content - pull secret + no_log: true + ansible.builtin.copy: + content: "{{ cifmw_manage_secrets_pullsecret_content }}" + dest: "{{ cifmw_manage_secrets_pullsecret_file }}" + mode: "0640" + +- name: Install required packages + ansible.builtin.include_role: + name: ci_setup + tasks_from: packages.yml diff --git a/roles/devscripts_sno/tasks/prepare_qemu_home_access.yml b/roles/devscripts_sno/tasks/prepare_qemu_home_access.yml new file mode 100644 index 000000000..f119da0ab --- /dev/null +++ b/roles/devscripts_sno/tasks/prepare_qemu_home_access.yml @@ -0,0 +1,37 @@ +--- +# Copyright Red Hat, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# Libvirt runs VMs as user qemu. Agent ISOs under the playbook user's home +# (mode 0700) are not readable without traverse ACL — same pattern as +# libvirt_manager/tasks/virsh_checks.yml. +# More info: https://github.com/openshift-metal3/dev-scripts/tree/master/agent#troubleshooting + +- name: Ensure packages for qemu user and setfacl + become: true + ansible.builtin.package: + name: + - acl + - qemu-kvm + state: present + +- name: Allow QEMU to traverse the user home directory + become: true + ansible.posix.acl: + path: "{{ ansible_user_dir }}" + entity: qemu + etype: user + permissions: rx + state: present diff --git a/roles/devscripts_sno/tasks/setup.yml b/roles/devscripts_sno/tasks/setup.yml new file mode 100644 index 000000000..6acfacec9 --- /dev/null +++ b/roles/devscripts_sno/tasks/setup.yml @@ -0,0 +1,16 @@ +--- +- name: Run devscripts role - build config + ansible.builtin.include_role: + name: devscripts + tasks_from: build_config.yml + +- name: Run devscripts role - 100_pre.yml + ansible.builtin.include_role: + name: devscripts + tasks_from: 100_pre.yml + +- name: Run devscripts make agent + cifmw.general.ci_script: + chdir: "{{ cifmw_devscripts_sno_repo_dir }}" + output_dir: "{{ cifmw_devscripts_sno_artifacts_dir }}" + script: "timeout {{ cifmw_devscripts_sno_installer_timeout }} make agent" diff --git a/scenarios/reproducers/va-hci-minimal-sno.yml b/scenarios/reproducers/va-hci-minimal-sno.yml new file mode 100644 index 000000000..fff1ac1a4 --- /dev/null +++ b/scenarios/reproducers/va-hci-minimal-sno.yml @@ -0,0 +1,29 @@ +--- +cifmw_parent_scenario: "scenarios/reproducers/va-hci-base.yml" + +cifmw_devscripts_sno: true +cifmw_reproducer_allow_one_ocp: true + +# Force use CPU passtrhough in OCP to avoid performance degradation +# when nested virtualization was used. +cifmw_devscripts_cpu_passthrough: true + +## devscript support for OCP deploy +# Libvirt ocpbm gateway (must match networks.ocpbm address 192.168.111.1 above). +# Required when manage_br_bridge is "n"; otherwise 135_patch_src uses the +# hypervisor management IP and libvirt rejects the DHCP range on 192.168.111.x. +cifmw_devscripts_host_bm_net_ip_addr: "192.168.111.1" + +cifmw_devscripts_config_overrides: + fips_mode: "{{ cifmw_fips_enabled | default(false) | bool }}" + num_masters: 1 + master_memory: 16384 + master_disk: 120 + master_vcpu: 12 + external_subnet_v4: 192.168.111.0/24 + ip_stack: v4 + agent_e2e_test_scenario: SNO_IPV4 + agent_platform_type: none + openshift_release_type: ga + openshift_version: stable-4.18 + ntp_servers: pool.ntp.org diff --git a/zuul.d/molecule.yaml b/zuul.d/molecule.yaml index 903d2e114..84a9fee42 100644 --- a/zuul.d/molecule.yaml +++ b/zuul.d/molecule.yaml @@ -965,6 +965,15 @@ - ^.config/molecule/.* name: cifmw-molecule-deploy_minio parent: cifmw-molecule-noop +- job: + files: + - ^common-requirements.txt + - ^test-requirements.txt + - ^roles/devscripts_sno/.* + - ^ci/playbooks/molecule.* + - ^.config/molecule/.* + name: cifmw-molecule-devscripts_sno + parent: cifmw-molecule-noop - job: files: - ^common-requirements.txt diff --git a/zuul.d/projects.yaml b/zuul.d/projects.yaml index 9b4b8e84b..f7f8ef64f 100644 --- a/zuul.d/projects.yaml +++ b/zuul.d/projects.yaml @@ -47,6 +47,7 @@ - cifmw-molecule-deploy_loki - cifmw-molecule-deploy_minio - cifmw-molecule-devscripts + - cifmw-molecule-devscripts_sno - cifmw-molecule-discover_latest_image - cifmw-molecule-dlrn_promote - cifmw-molecule-dlrn_report