M #-: Add Open vSwitch role

playbooks/pre.yml

@@ -9,6 +9,8 @@
       become: false
       tags: [bastion]
 
+# Cluster integrity / consistency.
+
 - hosts:
     - "{{ router_group | d('router') }}"
     - "{{ frontend_group | d('frontend') }}"
@@ -21,10 +23,13 @@
   collections:
     - opennebula.deploy
   roles:
-    - role: helper/python3
+    - role: helper/python3 # for python intepreter
+
     - role: helper/facts
+
     - role: helper/cache
-    - role: helper/python3
+
+    - role: helper/python3 # for extra OS/PyPI packages
 
 - hosts:
     - "{{ frontend_group | d('frontend') }}"
@@ -46,6 +51,7 @@
   roles:
     - role: helper/facts
       tags: [always]
+
     - role: precheck/pre_reboot
 
 - hosts:
@@ -70,3 +76,27 @@
       tags: [kernel]
 
     - role: precheck/post_reboot
+
+# Early PCI/SR-IOV + OVS/DPDK management.
+
+- hosts:
+    - "{{ node_group | d('node') }}"
+  collections:
+    - opennebula.deploy
+  roles:
+    # NOTE: PCI/SR-IOV management is limited to HV nodes exclusively.
+    - role: helper/pci
+      tags: [pci, pci_passthrough]
+
+- hosts:
+    - "{{ frontend_group | d('frontend') }}"
+    - "{{ node_group | d('node') }}"
+  collections:
+    - opennebula.deploy
+  roles:
+    # NOTE: This is required here since OVS/DPDK may require custom package sources.
+    - role: repository
+      tags: [openvswitch]
+
+    - role: openvswitch
+      tags: [openvswitch]

playbooks/site.yml

@@ -10,11 +10,6 @@
     # Make sure all facts are usable on the playbook level.
     - role: common
 
-    # The repository role can be safely executed on the whole inventory, it will auto-detect
-    # which types of repositories it should install on which hosts.
-    - role: repository
-      tags: [preinstall, prometheus]
-
 - hosts: "{{ frontend_group | d('frontend') }}"
   tags: [frontend, stage1]
   collections:

roles/helper/pci/tasks/query.yml

@@ -160,19 +160,31 @@
         {{- output.append(_facts[v].slaves | d([_facts[v].device | d(none)] | select)) -}}
       {%- endfor -%}
       {{- output | flatten -}}
+    # Do also a quick crosscheck with OVS/DPDK config (when available).
+    _dpdk_pci_addrs: >-
+      {{ (ovs.iface | d({})).values() | map(attribute='set', default=[])
+                                      | map('selectattr', 'options:dpdk-devargs', 'defined')
+                                      | select
+                                      | flatten
+                                      | map(attribute='options:dpdk-devargs') }}
+    _dpdk_interfaces: >-
+      {{ (ovs.iface | d({})).keys() | intersect(_facts.interfaces) }}
   block:
     - name: Query udev for device info
       ansible.builtin.command:
         cmd: "udevadm info --query=property --property=ID_PATH --value {{ _paths | join(' ') }}"
       vars:
         _paths: >-
-          {{ _interfaces | map('regex_replace', '^(.*)$', "-p '/sys/class/net/\g<1>'") }}
+          {{ (_interfaces + _dpdk_interfaces) | unique
+                                              | map('regex_replace', '^(.*)$', "-p '/sys/class/net/\g<1>'") }}
       register: command_udevadm_info
       changed_when: false
 
     - name: Gather forbidden PCI addresses
       ansible.builtin.set_fact:
-        pci_forbidden_addresses: >-
+        pci_forbidden_addresses: "{{ (_pci_addrs + _dpdk_pci_addrs) | unique }}"
+      vars:
+        _pci_addrs: >-
           {{ command_udevadm_info.stdout_lines | select
                                                | map('regex_replace', '^pci-', '') }}
 
@@ -186,6 +198,7 @@
     fail_msg: >-
       Forbidden PCI addresses {{ _detected }} detected, aborting!
       Please adjust 'pci_devices' to exclude forbidden PCI addresses.
+      You might also want to look for conflicts with OVS/DPDK config.
   vars:
     _detected: >-
       {{ lspci_devices | map(attribute='Slot') | intersect(pci_forbidden_addresses) }}

roles/openvswitch/README.md

@@ -0,0 +1,112 @@
+Role: opennebula.deploy.openvswitch
+===================================
+
+A role that **replaces** OS default networking with OVS/DPDK.
+
+Requirements
+------------
+
+N/A
+
+Role Variables
+--------------
+
+| Name           | Type   | Default               | Example       | Description                              |
+|----------------|--------|-----------------------|---------------|------------------------------------------|
+| `ovs`          | `dict` | (check role defaults) | (check below) | OVS/DPDK config.                         |
+| `ovs_packages` | `dict` | (check role defaults) |               | OVS/DPDK packages grouped per OS distro. |
+
+Dependencies
+------------
+
+N/A
+
+Example Playbook
+----------------
+
+    - hosts: node
+      vars:
+        kernel_ok_to_reboot: true
+        kernel_params:
+          - default_hugepagesz: "1G"
+          - hugepagesz: "1G"
+          - hugepages: 3
+          - intel_iommu: "on"
+        kernel_modules:
+          - load: vfio-pci
+          - load: vfio_iommu_type1
+            options: ["allow_unsafe_interrupts=1"] # for virtio-net-pci devices
+        opennebula_repo_pre_enable:
+          AlmaLinux:
+            extra_rpms:
+              '10': [centos-release-nfv-openvswitch]
+            config_manager:
+              '10': [crb, epel, highavailability, centos-nfv-openvswitch]
+          RedHat:
+            subscription_manager:
+              '9':
+                - codeready-builder-for-rhel-9-x86_64-rpms
+                - rhel-9-for-x86_64-highavailability-rpms
+                - fast-datapath-for-rhel-9-x86_64-rpms
+        ovs:
+          set:
+            - other_config:dpdk-init: 'true'
+            - other_config:dpdk-socket-mem: '1024,0'
+          port:
+            ovsbr0: # "internal" port
+              set:
+                - tag: 123
+          iface:
+            ovsbr0: # "internal" port
+              set:
+                - mtu_request: 1500
+            dpdk-p0:
+              set:
+                - type: dpdkvhostuserclient
+                - mtu_request: 9000
+                - options:vhost-server-path: /var/tmp/dpdk-p0
+            dpdk-p1:
+              set:
+                - type: dpdk
+                - mtu_request: 9000
+                - options:dpdk-devargs: '0000:02:00.0'
+              driver: vfio-pci # this is the default
+            dpdk-p2:
+              set:
+                - type: dpdk
+                - mtu_request: 9000
+                - options:dpdk-devargs: '0000:03:00.0'
+              driver: omit # skip forcing the driver
+            eth3: {} # non-DPDK device
+          bond:
+            bond0:
+              ifaces: [dpdk-p1, dpdk-p2]
+              set:
+                - bond_mode: active-backup
+          br:
+            ovsbr0:
+              ports: [dpdk-p0, bond0]
+              set:
+                - datapath_type: netdev
+              addrs:
+                - cidr: "{{ ansible_default_ipv4.address ~ '/' ~ ansible_default_ipv4.prefix }}"
+                  metric: 400
+              gw: "{{ ansible_default_ipv4.gateway }}"
+              dns: ["{{ ansible_default_ipv4.gateway }}"]
+            ovsbr1: # non-DPDK bridge
+              ports: [eth3]
+      roles:
+        - role: opennebula.deploy.helper.facts
+        - role: opennebula.deploy.helper.kernel
+        - role: opennebula.deploy.repository
+        - role: opennebula.deploy.openvswitch
+
+License
+-------
+
+Apache-2.0
+
+Author Information
+------------------
+
+[OpenNebula Systems](https://opennebula.io/)

roles/openvswitch/defaults/main.yml

@@ -0,0 +1,61 @@
+---
+ovs_defaults:
+  iface: {}
+  bond: {}
+  br: {}
+  port: {}
+
+ovs_packages:
+  AlmaLinux:
+    - ethtool
+    - iproute
+    - iputils
+    - openvswitch3.5
+    - systemd-resolved
+  Debian:
+    - ethtool
+    - iproute2
+    - iputils-arping
+    - openvswitch-switch
+    - systemd-resolved
+  RedHat:
+    - ethtool
+    - iproute
+    - iputils
+    - openvswitch3.6
+    - systemd-resolved
+  Suse:
+    - ethtool
+    - iproute2
+    - iputils
+    - openvswitch
+    - systemd-resolved
+
+ovs_packages_dpdk:
+  AlmaLinux:
+    - dpdk-tools
+    - ethtool
+    - iproute
+    - iputils
+    - openvswitch3.5
+    - systemd-resolved
+  Debian:
+    - ethtool
+    - iproute2
+    - iputils-arping
+    - openvswitch-switch-dpdk
+    - systemd-resolved
+  RedHat:
+    - dpdk-tools
+    - ethtool
+    - iproute
+    - iputils
+    - openvswitch3.6
+    - systemd-resolved
+  Suse:
+    - dpdk-tools
+    - ethtool
+    - iproute2
+    - iputils
+    - openvswitch
+    - systemd-resolved

roles/openvswitch/meta/main.yml

@@ -0,0 +1,3 @@
+---
+collections:
+  - opennebula.deploy